From 937973bce28f7345945a9bd77e4ed02f62c54d36 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Wed, 16 May 2018 22:54:23 +0800 Subject: [PATCH 01/79] add cf no_normalization --- .../methods/cf/normalization/CMakeLists.txt | 14 ++++ .../cf/normalization/no_normalization.hpp | 82 +++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 src/mlpack/methods/cf/normalization/CMakeLists.txt create mode 100644 src/mlpack/methods/cf/normalization/no_normalization.hpp diff --git a/src/mlpack/methods/cf/normalization/CMakeLists.txt b/src/mlpack/methods/cf/normalization/CMakeLists.txt new file mode 100644 index 0000000000..aaf2fb4c5e --- /dev/null +++ b/src/mlpack/methods/cf/normalization/CMakeLists.txt @@ -0,0 +1,14 @@ +# Define the files we need to compile +# Anything not in this list will not be compiled into mlpack. +set(SOURCES + no_normalization.hpp +) + +# Add directory name to sources. +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() +# Append sources (with directory name) to list of all mlpack sources (used at +# the parent scope). +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/methods/cf/normalization/no_normalization.hpp b/src/mlpack/methods/cf/normalization/no_normalization.hpp new file mode 100644 index 0000000000..41e5a94816 --- /dev/null +++ b/src/mlpack/methods/cf/normalization/no_normalization.hpp @@ -0,0 +1,82 @@ +/** + * @file no_normalization.hpp + * @author Wenhao Huang + * + * This class performs no normalization. It is used as default type of + * normalization for CF class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_CF_NORMALIZATION_NO_NORMALIZATION_HPP +#define MLPACK_METHODS_CF_NORMALIZATION_NO_NORMALIZATION_HPP + +#include + +namespace mlpack { +namespace cf { + +/** + * This normalization class doesn't perform any normalization. It is the default + * normalization type for CF class. + */ +class NoNormalization +{ + public: + // Empty constructor. + NoNormalization(const arma::mat& /* data */) { } + + // Empty constructor. + NoNormalization(const arma::sp_mat* /* cleanedData */) { } + + /** + * Do nothing. + * + * @param data Input dataset in the form of coordinate list. + */ + inline void Normalize(const arma::mat& /* data */) const { } + + /** + * Do nothing. + * + * @param cleanedData Sparse matrix data. + */ + inline void Normalize(const arma::mat& /* cleanedData */) const { } + + /** + * Do nothing. + * + * @param user User ID. + * @param item Item ID. + * @param rating Computed rating before denormalization. + */ + inline double Denormalize(const int /* user */, + const int /* item */, + const double rating) const + { + return rating; + } + + /** + * Do nothing. + * + * @param combinations User/Item combinations. + * @param predictions Predicted Ratings for each User/Item combination. + */ + inline void Denormalize(const arma::Mat& /* combinations */, + const arma::vec& /* predictions */) const + { } + + /** + * Serialization. + */ + template + void serialize(Archive& /* ar */, const unsigned int /* version */) { } +}; + +} // namespace cf +} // namespace mlpack + +#endif From 6cad42e6ffe1383ab4d0b4b305f434acab257d44 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Thu, 17 May 2018 00:08:44 +0800 Subject: [PATCH 02/79] normalizationType constructor --- src/mlpack/methods/cf/normalization/no_normalization.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/mlpack/methods/cf/normalization/no_normalization.hpp b/src/mlpack/methods/cf/normalization/no_normalization.hpp index 41e5a94816..75575124cd 100644 --- a/src/mlpack/methods/cf/normalization/no_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/no_normalization.hpp @@ -26,10 +26,7 @@ class NoNormalization { public: // Empty constructor. - NoNormalization(const arma::mat& /* data */) { } - - // Empty constructor. - NoNormalization(const arma::sp_mat* /* cleanedData */) { } + NoNormalization() { } /** * Do nothing. From ea6d6914ddd97628b9b97be660892072c3a28e3a Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Thu, 17 May 2018 20:14:06 +0800 Subject: [PATCH 03/79] small bugfix --- src/mlpack/methods/cf/normalization/no_normalization.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/cf/normalization/no_normalization.hpp b/src/mlpack/methods/cf/normalization/no_normalization.hpp index 75575124cd..30bb09dad2 100644 --- a/src/mlpack/methods/cf/normalization/no_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/no_normalization.hpp @@ -40,7 +40,7 @@ class NoNormalization * * @param cleanedData Sparse matrix data. */ - inline void Normalize(const arma::mat& /* cleanedData */) const { } + inline void Normalize(const arma::sp_mat& /* cleanedData */) const { } /** * Do nothing. From 0630ed345112ff0bdba25015431cf78b5a6815c7 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Thu, 17 May 2018 20:14:37 +0800 Subject: [PATCH 04/79] add normalization cmakelist --- src/mlpack/methods/cf/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mlpack/methods/cf/CMakeLists.txt b/src/mlpack/methods/cf/CMakeLists.txt index 01796f29da..e4c8f7fe23 100644 --- a/src/mlpack/methods/cf/CMakeLists.txt +++ b/src/mlpack/methods/cf/CMakeLists.txt @@ -3,7 +3,6 @@ set(SOURCES cf.hpp cf_impl.hpp - cf.cpp svd_wrapper.hpp svd_wrapper_impl.hpp ) From cc1494fbac3cc5dffe698e2e25d5b43aa58ddcf2 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Thu, 17 May 2018 20:15:16 +0800 Subject: [PATCH 05/79] modify cf files --- src/mlpack/methods/cf/cf.cpp | 267 --------------------------- src/mlpack/methods/cf/cf.hpp | 12 +- src/mlpack/methods/cf/cf_impl.hpp | 294 +++++++++++++++++++++++++++++- src/mlpack/methods/cf/cf_main.cpp | 16 +- 4 files changed, 304 insertions(+), 285 deletions(-) delete mode 100644 src/mlpack/methods/cf/cf.cpp diff --git a/src/mlpack/methods/cf/cf.cpp b/src/mlpack/methods/cf/cf.cpp deleted file mode 100644 index e092004dd9..0000000000 --- a/src/mlpack/methods/cf/cf.cpp +++ /dev/null @@ -1,267 +0,0 @@ -/** - * @file cf.cpp - * @author Mudit Raj Gupta - * @author Sumedh Ghaisas - * - * Collaborative Filtering. - * - * Implementation of CF class to perform Collaborative Filtering on the - * specified data set. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include "cf.hpp" - -#include - -namespace mlpack { -namespace cf { - -// Default CF constructor. -CF::CF(const size_t numUsersForSimilarity, - const size_t rank) : - numUsersForSimilarity(numUsersForSimilarity), - rank(rank) -{ - // Validate neighbourhood size. - if (numUsersForSimilarity < 1) - { - Log::Warn << "CF::CF(): neighbourhood size should be > 0 (" - << numUsersForSimilarity << " given). Setting value to 5.\n"; - // Set default value of 5. - this->numUsersForSimilarity = 5; - } -} - -void CF::GetRecommendations(const size_t numRecs, - arma::Mat& recommendations) -{ - // Generate list of users. Maybe it would be more efficient to pass an empty - // users list, and then have the other overload of GetRecommendations() assume - // that if users is empty, then recommendations should be generated for all - // users? - arma::Col users = arma::linspace >(0, - cleanedData.n_cols - 1, cleanedData.n_cols); - - // Call the main overload for recommendations. - GetRecommendations(numRecs, recommendations, users); -} - -void CF::GetRecommendations(const size_t numRecs, - arma::Mat& recommendations, - const arma::Col& users) -{ - // We want to avoid calculating the full rating matrix, so we will do nearest - // neighbor search only on the H matrix, using the observation that if the - // rating matrix X = W*H, then d(X.col(i), X.col(j)) = d(W H.col(i), W - // H.col(j)). This can be seen as nearest neighbor search on the H matrix - // with the Mahalanobis distance where M^{-1} = W^T W. So, we'll decompose - // M^{-1} = L L^T (the Cholesky decomposition), and then multiply H by L^T. - // Then we can perform nearest neighbor search. - arma::mat l = arma::chol(w.t() * w); - arma::mat stretchedH = l * h; // Due to the Armadillo API, l is L^T. - - // Now, we will use the decomposed w and h matrices to estimate what the user - // would have rated items as, and then pick the best items. - - // Temporarily store feature vector of queried users. - arma::mat query(stretchedH.n_rows, users.n_elem); - - // Select feature vectors of queried users. - for (size_t i = 0; i < users.n_elem; i++) - query.col(i) = stretchedH.col(users(i)); - - // Temporary storage for neighborhood of the queried users. - arma::Mat neighborhood; - - // Calculate the neighborhood of the queried users. Note that the query user - // is part of the neighborhood---this is intentional. We want to use an - // average of both the query user and the local neighborhood of the query - // user. - // The neighbor search technique should be a template parameter. - neighbor::KNN a(stretchedH); - arma::mat resultingDistances; // Temporary storage. - a.Search(query, numUsersForSimilarity, neighborhood, resultingDistances); - - // Generate recommendations for each query user by finding the maximum numRecs - // elements in the averages matrix. - recommendations.set_size(numRecs, users.n_elem); - arma::mat values(numRecs, users.n_elem); - recommendations.fill(SIZE_MAX); - values.fill(DBL_MAX); - - for (size_t i = 0; i < users.n_elem; i++) - { - // First, calculate average of neighborhood values. - arma::vec averages; - averages.zeros(cleanedData.n_rows); - - for (size_t j = 0; j < neighborhood.n_rows; ++j) - averages += w * h.col(neighborhood(j, i)); - averages /= neighborhood.n_rows; - - // Let's build the list of candidate recomendations for the given user. - // Default candidate: the smallest possible value and invalid item number. - const Candidate def = std::make_pair(-DBL_MAX, cleanedData.n_rows); - std::vector vect(numRecs, def); - typedef std::priority_queue, CandidateCmp> - CandidateList; - CandidateList pqueue(CandidateCmp(), std::move(vect)); - - // Look through the averages column corresponding to the current user. - for (size_t j = 0; j < averages.n_rows; ++j) - { - // Ensure that the user hasn't already rated the item. - if (cleanedData(j, users(i)) != 0.0) - continue; // The user already rated the item. - - // Is the estimated value better than the worst candidate? - if (averages[j] > pqueue.top().first) - { - Candidate c = std::make_pair(averages[j], j); - pqueue.pop(); - pqueue.push(c); - } - } - - for (size_t p = 1; p <= numRecs; p++) - { - recommendations(numRecs - p, i) = pqueue.top().second; - values(numRecs - p, i) = pqueue.top().first; - pqueue.pop(); - } - - // If we were not able to come up with enough recommendations, issue a - // warning. - if (recommendations(numRecs - 1, i) == def.second) - Log::Warn << "Could not provide " << numRecs << " recommendations " - << "for user " << users(i) << " (not enough un-rated items)!" - << std::endl; - } -} - -// Predict the rating for a single user/item combination. -double CF::Predict(const size_t user, const size_t item) const -{ - // First, we need to find the nearest neighbors of the given user. - // We'll use the same technique as for GetRecommendations(). - - // We want to avoid calculating the full rating matrix, so we will do nearest - // neighbor search only on the H matrix, using the observation that if the - // rating matrix X = W*H, then d(X.col(i), X.col(j)) = d(W H.col(i), W - // H.col(j)). This can be seen as nearest neighbor search on the H matrix - // with the Mahalanobis distance where M^{-1} = W^T W. So, we'll decompose - // M^{-1} = L L^T (the Cholesky decomposition), and then multiply H by L^T. - // Then we can perform nearest neighbor search. - arma::mat l = arma::chol(w.t() * w); - arma::mat stretchedH = l * h; // Due to the Armadillo API, l is L^T. - - // Now, we will use the decomposed w and h matrices to estimate what the user - // would have rated items as, and then pick the best items. - - // Temporarily store feature vector of queried users. - arma::mat query = stretchedH.col(user); - - // Temporary storage for neighborhood of the queried users. - arma::Mat neighborhood; - - // Calculate the neighborhood of the queried users. - // This should be a templatized option. - neighbor::KNN a(stretchedH, neighbor::SINGLE_TREE_MODE); - arma::mat resultingDistances; // Temporary storage. - - a.Search(query, numUsersForSimilarity, neighborhood, resultingDistances); - - double rating = 0; // We'll take the average of neighborhood values. - - for (size_t j = 0; j < neighborhood.n_rows; ++j) - rating += arma::as_scalar(w.row(item) * h.col(neighborhood(j, 0))); - rating /= neighborhood.n_rows; - - return rating; -} - -// Predict the rating for a group of user/item combinations. -void CF::Predict(const arma::Mat& combinations, - arma::vec& predictions) const -{ - // First, for nearest neighbor search, stretch the H matrix. - arma::mat l = arma::chol(w.t() * w); - arma::mat stretchedH = l * h; // Due to the Armadillo API, l is L^T. - - // Now, we must determine those query indices we need to find the nearest - // neighbors for. This is easiest if we just sort the combinations matrix. - arma::Mat sortedCombinations(combinations.n_rows, - combinations.n_cols); - arma::uvec ordering = arma::sort_index(combinations.row(0).t()); - for (size_t i = 0; i < ordering.n_elem; ++i) - sortedCombinations.col(i) = combinations.col(ordering[i]); - - // Now, we have to get the list of unique users we will be searching for. - arma::Col users = arma::unique(combinations.row(0).t()); - - // Assemble our query matrix from the stretchedH matrix. - arma::mat queries(stretchedH.n_rows, users.n_elem); - for (size_t i = 0; i < queries.n_cols; ++i) - queries.col(i) = stretchedH.col(users[i]); - - // Now calculate the neighborhood of these users. - neighbor::KNN a(stretchedH); - arma::mat distances; - arma::Mat neighborhood; - - a.Search(queries, numUsersForSimilarity, neighborhood, distances); - - // Now that we have the neighborhoods we need, calculate the predictions. - predictions.set_size(combinations.n_cols); - - size_t user = 0; // Cumulative user count, because we are doing it in order. - for (size_t i = 0; i < sortedCombinations.n_cols; ++i) - { - // Could this be made faster by calculating dot products for multiple items - // at once? - double rating = 0.0; - - // Map the combination's user to the user ID used for kNN. - while (users[user] < sortedCombinations(0, i)) - ++user; - - for (size_t j = 0; j < neighborhood.n_rows; ++j) - rating += arma::as_scalar(w.row(sortedCombinations(1, i)) * - h.col(neighborhood(j, user))); - rating /= neighborhood.n_rows; - - predictions(ordering[i]) = rating; - } -} - -void CF::CleanData(const arma::mat& data, arma::sp_mat& cleanedData) -{ - // Generate list of locations for batch insert constructor for sparse - // matrices. - arma::umat locations(2, data.n_cols); - arma::vec values(data.n_cols); - for (size_t i = 0; i < data.n_cols; ++i) - { - // We have to transpose it because items are rows, and users are columns. - locations(1, i) = ((arma::uword) data(0, i)); - locations(0, i) = ((arma::uword) data(1, i)); - values(i) = data(2, i); - if (values(i) == 0) - Log::Warn << "User rating of 0 ignored for user " << locations(1, i) - << ", item " << locations(0, i) << "." << std::endl; - } - - // Find maximum user and item IDs. - const size_t maxItemID = (size_t) max(locations.row(0)) + 1; - const size_t maxUserID = (size_t) max(locations.row(1)) + 1; - - // Fill sparse matrix. - cleanedData = arma::sp_mat(locations, values, maxItemID, maxUserID); -} - -} // namespace cf -} // namespace mlpack diff --git a/src/mlpack/methods/cf/cf.hpp b/src/mlpack/methods/cf/cf.hpp index 1e0d01193a..62ce7e9672 100644 --- a/src/mlpack/methods/cf/cf.hpp +++ b/src/mlpack/methods/cf/cf.hpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -77,6 +78,8 @@ struct FactorizerTraits * the rating matrix (a W and H matrix). This must implement the method * Apply(arma::sp_mat& data, size_t rank, arma::mat& W, arma::mat& H). */ + +template class CF { public: @@ -85,7 +88,8 @@ class CF * call Train() before calling GetRecommendations() or any other functions! */ CF(const size_t numUsersForSimilarity = 5, - const size_t rank = 0); + const size_t rank = 0, + const NormalizationType normalization = NormalizationType()); /** * Initialize the CF object using an instantiated factorizer, immediately @@ -107,7 +111,8 @@ class CF CF(const arma::mat& data, FactorizerType factorizer = FactorizerType(), const size_t numUsersForSimilarity = 5, - const size_t rank = 0); + const size_t rank = 0, + const NormalizationType normalization = NormalizationType()); /** * Initialize the CF object using an instantiated factorizer, immediately @@ -132,6 +137,7 @@ class CF FactorizerType factorizer = FactorizerType(), const size_t numUsersForSimilarity = 5, const size_t rank = 0, + const NormalizationType normalization = NormalizationType(), const typename std::enable_if_t< !FactorizerTraits::UsesCoordinateList>* = 0); @@ -261,6 +267,8 @@ class CF arma::mat h; //! Cleaned data matrix. arma::sp_mat cleanedData; + //! Data normalization object. + NormalizationType normalization; //! Candidate represents a possible recommendation (value, item). typedef std::pair Candidate; diff --git a/src/mlpack/methods/cf/cf_impl.hpp b/src/mlpack/methods/cf/cf_impl.hpp index 2d552b7734..f859f3f8d9 100644 --- a/src/mlpack/methods/cf/cf_impl.hpp +++ b/src/mlpack/methods/cf/cf_impl.hpp @@ -53,13 +53,16 @@ void ApplyFactorizer(FactorizerType& factorizer, /** * Construct the CF object using an instantiated factorizer. */ +template template -CF::CF(const arma::mat& data, +CF::CF(const arma::mat& data, FactorizerType factorizer, const size_t numUsersForSimilarity, - const size_t rank) : + const size_t rank, + const NormalizationType normalization) : numUsersForSimilarity(numUsersForSimilarity), - rank(rank) + rank(rank), + normalization(normalization) { // Validate neighbourhood size. if (numUsersForSimilarity < 1) @@ -70,21 +73,26 @@ CF::CF(const arma::mat& data, this->numUsersForSimilarity = 5; } + // Normalize data and train. + normalization.Normalize(data); Train(data, factorizer); } /** * Construct the CF object using an instantiated factorizer. */ +template template -CF::CF(const arma::sp_mat& data, +CF::CF(const arma::sp_mat& data, FactorizerType factorizer, const size_t numUsersForSimilarity, const size_t rank, + const NormalizationType normalization, const typename std::enable_if_t< !FactorizerTraits::UsesCoordinateList>*) : numUsersForSimilarity(numUsersForSimilarity), - rank(rank) + rank(rank), + normalization(normalization) { // Validate neighbourhood size. if (numUsersForSimilarity < 1) @@ -95,11 +103,14 @@ CF::CF(const arma::sp_mat& data, this->numUsersForSimilarity = 5; } + // Normalize data and train. + normalization.Normalize(data); Train(data, factorizer); } +template template -void CF::Train(const arma::mat& data, FactorizerType factorizer) +void CF::Train(const arma::mat& data, FactorizerType factorizer) { CleanData(data, cleanedData); @@ -125,8 +136,9 @@ void CF::Train(const arma::mat& data, FactorizerType factorizer) Timer::Stop("cf_factorization"); } +template template -void CF::Train(const arma::sp_mat& data, +void CF::Train(const arma::sp_mat& data, FactorizerType factorizer, const typename std::enable_if_t::UsesCoordinateList>*) @@ -154,8 +166,9 @@ void CF::Train(const arma::sp_mat& data, } //! Serialize the model. +template template -void CF::serialize(Archive& ar, const unsigned int /* version */) +void CF::serialize(Archive& ar, const unsigned int /* version */) { // This model is simple; just serialize all the members. No special handling // required. @@ -164,8 +177,273 @@ void CF::serialize(Archive& ar, const unsigned int /* version */) ar & BOOST_SERIALIZATION_NVP(w); ar & BOOST_SERIALIZATION_NVP(h); ar & BOOST_SERIALIZATION_NVP(cleanedData); + ar & BOOST_SERIALIZATION_NVP(normalization); } + + + +// Default CF constructor. +template +CF::CF(const size_t numUsersForSimilarity, + const size_t rank, + const NormalizationType normalization) : + numUsersForSimilarity(numUsersForSimilarity), + rank(rank), + normalization(normalization) +{ + // Validate neighbourhood size. + if (numUsersForSimilarity < 1) + { + Log::Warn << "CF::CF(): neighbourhood size should be > 0 (" + << numUsersForSimilarity << " given). Setting value to 5.\n"; + // Set default value of 5. + this->numUsersForSimilarity = 5; + } +} + +template +void CF::GetRecommendations(const size_t numRecs, + arma::Mat& recommendations) +{ + // Generate list of users. Maybe it would be more efficient to pass an empty + // users list, and then have the other overload of GetRecommendations() assume + // that if users is empty, then recommendations should be generated for all + // users? + arma::Col users = arma::linspace >(0, + cleanedData.n_cols - 1, cleanedData.n_cols); + + // Call the main overload for recommendations. + GetRecommendations(numRecs, recommendations, users); +} + +template +void CF::GetRecommendations(const size_t numRecs, + arma::Mat& recommendations, + const arma::Col& users) +{ + // We want to avoid calculating the full rating matrix, so we will do nearest + // neighbor search only on the H matrix, using the observation that if the + // rating matrix X = W*H, then d(X.col(i), X.col(j)) = d(W H.col(i), W + // H.col(j)). This can be seen as nearest neighbor search on the H matrix + // with the Mahalanobis distance where M^{-1} = W^T W. So, we'll decompose + // M^{-1} = L L^T (the Cholesky decomposition), and then multiply H by L^T. + // Then we can perform nearest neighbor search. + arma::mat l = arma::chol(w.t() * w); + arma::mat stretchedH = l * h; // Due to the Armadillo API, l is L^T. + + // Now, we will use the decomposed w and h matrices to estimate what the user + // would have rated items as, and then pick the best items. + + // Temporarily store feature vector of queried users. + arma::mat query(stretchedH.n_rows, users.n_elem); + + // Select feature vectors of queried users. + for (size_t i = 0; i < users.n_elem; i++) + query.col(i) = stretchedH.col(users(i)); + + // Temporary storage for neighborhood of the queried users. + arma::Mat neighborhood; + + // Calculate the neighborhood of the queried users. Note that the query user + // is part of the neighborhood---this is intentional. We want to use an + // average of both the query user and the local neighborhood of the query + // user. + // The neighbor search technique should be a template parameter. + neighbor::KNN a(stretchedH); + arma::mat resultingDistances; // Temporary storage. + a.Search(query, numUsersForSimilarity, neighborhood, resultingDistances); + + // Generate recommendations for each query user by finding the maximum numRecs + // elements in the averages matrix. + recommendations.set_size(numRecs, users.n_elem); + arma::mat values(numRecs, users.n_elem); + recommendations.fill(SIZE_MAX); + values.fill(DBL_MAX); + + for (size_t i = 0; i < users.n_elem; i++) + { + // First, calculate average of neighborhood values. + arma::vec averages; + averages.zeros(cleanedData.n_rows); + + for (size_t j = 0; j < neighborhood.n_rows; ++j) + averages += w * h.col(neighborhood(j, i)); + averages /= neighborhood.n_rows; + + // Let's build the list of candidate recomendations for the given user. + // Default candidate: the smallest possible value and invalid item number. + const Candidate def = std::make_pair(-DBL_MAX, cleanedData.n_rows); + std::vector vect(numRecs, def); + typedef std::priority_queue, CandidateCmp> + CandidateList; + CandidateList pqueue(CandidateCmp(), std::move(vect)); + + // Look through the averages column corresponding to the current user. + for (size_t j = 0; j < averages.n_rows; ++j) + { + // Ensure that the user hasn't already rated the item. + if (cleanedData(j, users(i)) != 0.0) + continue; // The user already rated the item. + + // Is the estimated value better than the worst candidate? + if (averages[j] > pqueue.top().first) + { + // Denormalize rating before comparation. + double realRating = normalization.Denormalize(i, j, averages[j]); + Candidate c = std::make_pair(realRating, j); + pqueue.pop(); + pqueue.push(c); + } + } + + for (size_t p = 1; p <= numRecs; p++) + { + recommendations(numRecs - p, i) = pqueue.top().second; + values(numRecs - p, i) = pqueue.top().first; + pqueue.pop(); + } + + // If we were not able to come up with enough recommendations, issue a + // warning. + if (recommendations(numRecs - 1, i) == def.second) + Log::Warn << "Could not provide " << numRecs << " recommendations " + << "for user " << users(i) << " (not enough un-rated items)!" + << std::endl; + } +} + +// Predict the rating for a single user/item combination. +template +double CF::Predict(const size_t user, const size_t item) const +{ + // First, we need to find the nearest neighbors of the given user. + // We'll use the same technique as for GetRecommendations(). + + // We want to avoid calculating the full rating matrix, so we will do nearest + // neighbor search only on the H matrix, using the observation that if the + // rating matrix X = W*H, then d(X.col(i), X.col(j)) = d(W H.col(i), W + // H.col(j)). This can be seen as nearest neighbor search on the H matrix + // with the Mahalanobis distance where M^{-1} = W^T W. So, we'll decompose + // M^{-1} = L L^T (the Cholesky decomposition), and then multiply H by L^T. + // Then we can perform nearest neighbor search. + arma::mat l = arma::chol(w.t() * w); + arma::mat stretchedH = l * h; // Due to the Armadillo API, l is L^T. + + // Now, we will use the decomposed w and h matrices to estimate what the user + // would have rated items as, and then pick the best items. + + // Temporarily store feature vector of queried users. + arma::mat query = stretchedH.col(user); + + // Temporary storage for neighborhood of the queried users. + arma::Mat neighborhood; + + // Calculate the neighborhood of the queried users. + // This should be a templatized option. + neighbor::KNN a(stretchedH, neighbor::SINGLE_TREE_MODE); + arma::mat resultingDistances; // Temporary storage. + + a.Search(query, numUsersForSimilarity, neighborhood, resultingDistances); + + double rating = 0; // We'll take the average of neighborhood values. + + for (size_t j = 0; j < neighborhood.n_rows; ++j) + rating += arma::as_scalar(w.row(item) * h.col(neighborhood(j, 0))); + rating /= neighborhood.n_rows; + + // Denormalize rating and return. + double realRating = normalization.Denormalize(user, item, rating); + return realRating; +} + +// Predict the rating for a group of user/item combinations. +template +void CF::Predict(const arma::Mat& combinations, + arma::vec& predictions) const +{ + // First, for nearest neighbor search, stretch the H matrix. + arma::mat l = arma::chol(w.t() * w); + arma::mat stretchedH = l * h; // Due to the Armadillo API, l is L^T. + + // Now, we must determine those query indices we need to find the nearest + // neighbors for. This is easiest if we just sort the combinations matrix. + arma::Mat sortedCombinations(combinations.n_rows, + combinations.n_cols); + arma::uvec ordering = arma::sort_index(combinations.row(0).t()); + for (size_t i = 0; i < ordering.n_elem; ++i) + sortedCombinations.col(i) = combinations.col(ordering[i]); + + // Now, we have to get the list of unique users we will be searching for. + arma::Col users = arma::unique(combinations.row(0).t()); + + // Assemble our query matrix from the stretchedH matrix. + arma::mat queries(stretchedH.n_rows, users.n_elem); + for (size_t i = 0; i < queries.n_cols; ++i) + queries.col(i) = stretchedH.col(users[i]); + + // Now calculate the neighborhood of these users. + neighbor::KNN a(stretchedH); + arma::mat distances; + arma::Mat neighborhood; + + a.Search(queries, numUsersForSimilarity, neighborhood, distances); + + // Now that we have the neighborhoods we need, calculate the predictions. + predictions.set_size(combinations.n_cols); + + size_t user = 0; // Cumulative user count, because we are doing it in order. + for (size_t i = 0; i < sortedCombinations.n_cols; ++i) + { + // Could this be made faster by calculating dot products for multiple items + // at once? + double rating = 0.0; + + // Map the combination's user to the user ID used for kNN. + while (users[user] < sortedCombinations(0, i)) + ++user; + + for (size_t j = 0; j < neighborhood.n_rows; ++j) + rating += arma::as_scalar(w.row(sortedCombinations(1, i)) * + h.col(neighborhood(j, user))); + rating /= neighborhood.n_rows; + + predictions(ordering[i]) = rating; + } + + // Denormalize ratings. + normalization.Denormalize(combinations, predictions); +} + +template +void CF::CleanData(const arma::mat& data, arma::sp_mat& cleanedData) +{ + // Generate list of locations for batch insert constructor for sparse + // matrices. + arma::umat locations(2, data.n_cols); + arma::vec values(data.n_cols); + for (size_t i = 0; i < data.n_cols; ++i) + { + // We have to transpose it because items are rows, and users are columns. + locations(1, i) = ((arma::uword) data(0, i)); + locations(0, i) = ((arma::uword) data(1, i)); + values(i) = data(2, i); + if (values(i) == 0) + Log::Warn << "User rating of 0 ignored for user " << locations(1, i) + << ", item " << locations(0, i) << "." << std::endl; + } + + // Find maximum user and item IDs. + const size_t maxItemID = (size_t) max(locations.row(0)) + 1; + const size_t maxUserID = (size_t) max(locations.row(1)) + 1; + + // Fill sparse matrix. + cleanedData = arma::sp_mat(locations, values, maxItemID, maxUserID); +} + + + + } // namespace cf } // namespace mlpack diff --git a/src/mlpack/methods/cf/cf_main.cpp b/src/mlpack/methods/cf/cf_main.cpp index 2676975d10..72fe08ac9a 100644 --- a/src/mlpack/methods/cf/cf_main.cpp +++ b/src/mlpack/methods/cf/cf_main.cpp @@ -101,8 +101,8 @@ PARAM_DOUBLE_IN("min_residue", "Residue required to terminate the factorization" " (lower values generally mean better fits).", "r", 1e-5); // Load/save a model. -PARAM_MODEL_IN(CF, "input_model", "Trained CF model to load.", "m"); -PARAM_MODEL_OUT(CF, "output_model", "Output for trained CF model.", "M"); +PARAM_MODEL_IN(CF<>, "input_model", "Trained CF model to load.", "m"); +PARAM_MODEL_OUT(CF<>, "output_model", "Output for trained CF model.", "M"); // Query settings. PARAM_UMATRIX_IN("query", "List of query users for which recommendations should" @@ -116,7 +116,7 @@ PARAM_INT_IN("recommendations", "Number of recommendations to generate for each" PARAM_INT_IN("seed", "Set the random seed (0 uses std::time(NULL)).", "s", 0); -void ComputeRecommendations(CF* cf, +void ComputeRecommendations(CF<>* cf, const size_t numRecs, arma::Mat& recommendations) { @@ -142,7 +142,7 @@ void ComputeRecommendations(CF* cf, } } -void ComputeRMSE(CF* cf) +void ComputeRMSE(CF<>* cf) { // Now, compute each test point. arma::mat testData = std::move(CLI::GetParam("test")); @@ -169,7 +169,7 @@ void ComputeRMSE(CF* cf) Log::Info << "RMSE is " << rmse << "." << endl; } -void PerformAction(CF* c) +void PerformAction(CF<>* c) { if (CLI::HasParam("query") || CLI::HasParam("all_user_recommendations")) { @@ -187,7 +187,7 @@ void PerformAction(CF* c) if (CLI::HasParam("test")) ComputeRMSE(c); - CLI::GetParam("output_model") = c; + CLI::GetParam*>("output_model") = c; } template @@ -197,7 +197,7 @@ void PerformAction(Factorizer&& factorizer, { // Parameters for generating the CF object. const size_t neighborhood = (size_t) CLI::GetParam("neighborhood"); - CF* c = new CF(dataset, factorizer, neighborhood, rank); + CF<>* c = new CF<>(dataset, factorizer, neighborhood, rank); PerformAction(c); } @@ -348,7 +348,7 @@ static void mlpackMain() "test" }, true); // Load an input model. - CF* c = std::move(CLI::GetParam("input_model")); + CF<>* c = std::move(CLI::GetParam*>("input_model")); PerformAction(c); } From e0f9abb585ab8d93e728fdd2bbde15fa26f3ad15 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Thu, 17 May 2018 20:16:00 +0800 Subject: [PATCH 06/79] change CF to CF<> --- src/mlpack/tests/cf_test.cpp | 38 ++++++++++++------------- src/mlpack/tests/main_tests/cf_test.cpp | 22 +++++++------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index a7af46e4c2..7044849cc8 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -44,10 +44,10 @@ BOOST_AUTO_TEST_CASE(CFGetRecommendationsAllUsersTest) // Make data into sparse matrix. arma::sp_mat cleanedData; - CF::CleanData(dataset, cleanedData); + CF<>::CleanData(dataset, cleanedData); // Create a CF object. - CF c(cleanedData); + CF<> c(cleanedData); // Generate recommendations when query set is not specified. c.GetRecommendations(numRecs, recommendations); @@ -84,9 +84,9 @@ BOOST_AUTO_TEST_CASE(CFGetRecommendationsQueriedUserTest) // Make data into sparse matrix. arma::sp_mat cleanedData; - CF::CleanData(dataset, cleanedData); + CF<>::CleanData(dataset, cleanedData); - CF c(cleanedData); + CF<> c(cleanedData); // Generate recommendations when query set is specified. c.GetRecommendations(numRecsDefault, recommendations, users); @@ -143,10 +143,10 @@ BOOST_AUTO_TEST_CASE(RecommendationAccuracyTest) // Make data into sparse matrix. arma::sp_mat cleanedData; - CF::CleanData(dataset, cleanedData); + CF<>::CleanData(dataset, cleanedData); // Now create the CF object. - CF c(cleanedData); + CF<> c(cleanedData); // Obtain 150 recommendations for the users in savedCols, and make sure the // missing item shows up in most of them. First, create the list of users, @@ -237,10 +237,10 @@ BOOST_AUTO_TEST_CASE(CFPredictTest) // Make data into sparse matrix. arma::sp_mat cleanedData; - CF::CleanData(dataset, cleanedData); + CF<>::CleanData(dataset, cleanedData); // Now create the CF object. - CF c(cleanedData); + CF<> c(cleanedData); // Now, for each removed rating, make sure the prediction is... reasonably // accurate. @@ -304,10 +304,10 @@ BOOST_AUTO_TEST_CASE(CFBatchPredictTest) // Make data into sparse matrix. arma::sp_mat cleanedData; - CF::CleanData(dataset, cleanedData); + CF<>::CleanData(dataset, cleanedData); // Now create the CF object. - CF c(cleanedData); + CF<> c(cleanedData); // Get predictions for all user/item pairs we held back. arma::Mat combinations(2, savedCols.n_cols); @@ -335,7 +335,7 @@ BOOST_AUTO_TEST_CASE(TrainTest) // Generate random data. arma::sp_mat randomData; randomData.sprandu(100, 100, 0.3); - CF c(randomData); + CF<> c(randomData); // Now retrain with data we know about. arma::mat dataset; @@ -377,7 +377,7 @@ BOOST_AUTO_TEST_CASE(TrainTest) // Make data into sparse matrix. arma::sp_mat cleanedData; - CF::CleanData(dataset, cleanedData); + CF<>::CleanData(dataset, cleanedData); // Now retrain. c.Train(dataset); @@ -406,7 +406,7 @@ BOOST_AUTO_TEST_CASE(TrainTest) BOOST_AUTO_TEST_CASE(EmptyConstructorTrainTest) { // Use default constructor. - CF c; + CF<> c; // Now retrain with data we know about. arma::mat dataset; @@ -448,7 +448,7 @@ BOOST_AUTO_TEST_CASE(EmptyConstructorTrainTest) // Make data into sparse matrix. arma::sp_mat cleanedData; - CF::CleanData(dataset, cleanedData); + CF<>::CleanData(dataset, cleanedData); // Now retrain. c.Train(cleanedData); @@ -481,16 +481,16 @@ BOOST_AUTO_TEST_CASE(SerializationTest) data::Load("GroupLensSmall.csv", dataset); arma::sp_mat cleanedData; - CF::CleanData(dataset, cleanedData); + CF<>::CleanData(dataset, cleanedData); - CF c(cleanedData); + CF<> c(cleanedData); arma::sp_mat randomData; randomData.sprandu(100, 100, 0.3); - CF cXml(randomData); - CF cBinary; - CF cText(cleanedData, amf::NMFALSFactorizer(), 5, 5); + CF<> cXml(randomData); + CF<> cBinary; + CF<> cText(cleanedData, amf::NMFALSFactorizer(), 5, 5); SerializeObjectAll(c, cXml, cText, cBinary); diff --git a/src/mlpack/tests/main_tests/cf_test.cpp b/src/mlpack/tests/main_tests/cf_test.cpp index 4048bbb3d7..a0073edfa0 100644 --- a/src/mlpack/tests/main_tests/cf_test.cpp +++ b/src/mlpack/tests/main_tests/cf_test.cpp @@ -215,7 +215,7 @@ BOOST_AUTO_TEST_CASE(CFModelReuseTest) SetInputParam("query", std::move(query)); SetInputParam("recommendations", recommendations); SetInputParam("input_model", - std::move(CLI::GetParam("output_model"))); + std::move(CLI::GetParam*>("output_model"))); mlpackMain(); @@ -262,7 +262,7 @@ BOOST_AUTO_TEST_CASE(CFRankTest) mlpackMain(); - const CF* outputModel = CLI::GetParam("output_model"); + const CF<>* outputModel = CLI::GetParam*>("output_model"); BOOST_REQUIRE_EQUAL(outputModel->Rank(), rank); } @@ -274,7 +274,7 @@ BOOST_AUTO_TEST_CASE(CFMinResidueTest) { mat dataset; data::Load("GroupLensSmall.csv", dataset); - const CF* outputModel; + const CF<>* outputModel; // Set a larger min_residue. SetInputParam("min_residue", double(100)); @@ -286,7 +286,7 @@ BOOST_AUTO_TEST_CASE(CFMinResidueTest) mlpack::math::FixedRandomSeed(); mlpackMain(); - outputModel = CLI::GetParam("output_model"); + outputModel = CLI::GetParam*>("output_model"); const mat w1 = outputModel->W(); const mat h1 = outputModel->H(); @@ -302,7 +302,7 @@ BOOST_AUTO_TEST_CASE(CFMinResidueTest) mlpack::math::FixedRandomSeed(); mlpackMain(); - outputModel = CLI::GetParam("output_model"); + outputModel = CLI::GetParam*>("output_model"); const mat w2 = outputModel->W(); const mat h2 = outputModel->H(); @@ -317,7 +317,7 @@ BOOST_AUTO_TEST_CASE(CFIterationOnlyTerminationTest) { mat dataset; data::Load("GroupLensSmall.csv", dataset); - const CF* outputModel; + const CF<>* outputModel; // Set iteration_only_termination. SetInputParam("iteration_only_termination", true); @@ -329,7 +329,7 @@ BOOST_AUTO_TEST_CASE(CFIterationOnlyTerminationTest) mlpack::math::FixedRandomSeed(); mlpackMain(); - outputModel = CLI::GetParam("output_model"); + outputModel = CLI::GetParam*>("output_model"); const mat w1 = outputModel->W(); const mat h1 = outputModel->H(); @@ -344,7 +344,7 @@ BOOST_AUTO_TEST_CASE(CFIterationOnlyTerminationTest) mlpack::math::FixedRandomSeed(); mlpackMain(); - outputModel = CLI::GetParam("output_model"); + outputModel = CLI::GetParam*>("output_model"); const mat w2 = outputModel->W(); const mat h2 = outputModel->H(); @@ -359,7 +359,7 @@ BOOST_AUTO_TEST_CASE(CFMaxIterationsTest) { mat dataset; data::Load("GroupLensSmall.csv", dataset); - const CF* outputModel; + const CF<>* outputModel; // Set a larger max_iterations. SetInputParam("max_iterations", int(100)); @@ -370,7 +370,7 @@ BOOST_AUTO_TEST_CASE(CFMaxIterationsTest) mlpack::math::FixedRandomSeed(); mlpackMain(); - outputModel = CLI::GetParam("output_model"); + outputModel = CLI::GetParam*>("output_model"); const mat w1 = outputModel->W(); const mat h1 = outputModel->H(); @@ -385,7 +385,7 @@ BOOST_AUTO_TEST_CASE(CFMaxIterationsTest) mlpack::math::FixedRandomSeed(); mlpackMain(); - outputModel = CLI::GetParam("output_model"); + outputModel = CLI::GetParam*>("output_model"); const mat w2 = outputModel->W(); const mat h2 = outputModel->H(); From 40d399d98e2852f7ab9b7d0de4d892789a04883a Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Thu, 17 May 2018 22:43:20 +0800 Subject: [PATCH 07/79] add normalization to cmakelist --- src/mlpack/methods/cf/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mlpack/methods/cf/CMakeLists.txt b/src/mlpack/methods/cf/CMakeLists.txt index e4c8f7fe23..ce03161fd1 100644 --- a/src/mlpack/methods/cf/CMakeLists.txt +++ b/src/mlpack/methods/cf/CMakeLists.txt @@ -7,6 +7,8 @@ set(SOURCES svd_wrapper_impl.hpp ) +add_subdirectory(normalization) + # Add directory name to sources. set(DIR_SRCS) foreach(file ${SOURCES}) From 03fd3eb492083079c44618d12b7b08a460bb9db6 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Thu, 17 May 2018 22:45:29 +0800 Subject: [PATCH 08/79] update comments --- src/mlpack/methods/cf/cf.hpp | 8 ++- src/mlpack/methods/cf/cf_impl.hpp | 72 +++++++++---------- .../cf/normalization/no_normalization.hpp | 2 +- 3 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/mlpack/methods/cf/cf.hpp b/src/mlpack/methods/cf/cf.hpp index 62ce7e9672..5691ec974c 100644 --- a/src/mlpack/methods/cf/cf.hpp +++ b/src/mlpack/methods/cf/cf.hpp @@ -58,7 +58,7 @@ struct FactorizerTraits * extern arma::Col users; // users seeking recommendations * arma::Mat recommendations; // Recommendations * - * CF cf(data); // Default options. + * CF<> cf(data); // Default options. * * // Generate 10 recommendations for all users. * cf.GetRecommendations(10, recommendations); @@ -74,6 +74,10 @@ struct FactorizerTraits * are in a matrix that holds doubles, should hold integer (or size_t) values. * The user and item indices are assumed to start at 0. * + * @tparam NormalizationType The type of normalization performed on raw data. + * Data is normalized before calling Train() method. Predicted rating is + * denormalized before return. + * * @tparam FactorizerType The type of matrix factorization to use to decompose * the rating matrix (a W and H matrix). This must implement the method * Apply(arma::sp_mat& data, size_t rank, arma::mat& W, arma::mat& H). @@ -106,6 +110,7 @@ class CF * @param factorizer Instantiated factorizer object. * @param numUsersForSimilarity Size of the neighborhood. * @param rank Rank parameter for matrix factorization. + * @param normalization Instantiated normalization object. */ template CF(const arma::mat& data, @@ -131,6 +136,7 @@ class CF * @param factorizer Instantiated factorizer object. * @param numUsersForSimilarity Size of the neighborhood. * @param rank Rank parameter for matrix factorization. + * @param normalization Instantiated normalization object. */ template CF(const arma::sp_mat& data, diff --git a/src/mlpack/methods/cf/cf_impl.hpp b/src/mlpack/methods/cf/cf_impl.hpp index f859f3f8d9..170a905fa8 100644 --- a/src/mlpack/methods/cf/cf_impl.hpp +++ b/src/mlpack/methods/cf/cf_impl.hpp @@ -50,6 +50,25 @@ void ApplyFactorizer(FactorizerType& factorizer, factorizer.Apply(cleanedData, rank, w, h); } +// Default CF constructor. +template +CF::CF(const size_t numUsersForSimilarity, + const size_t rank, + const NormalizationType normalization) : + numUsersForSimilarity(numUsersForSimilarity), + rank(rank), + normalization(normalization) +{ + // Validate neighbourhood size. + if (numUsersForSimilarity < 1) + { + Log::Warn << "CF::CF(): neighbourhood size should be > 0 (" + << numUsersForSimilarity << " given). Setting value to 5.\n"; + // Set default value of 5. + this->numUsersForSimilarity = 5; + } +} + /** * Construct the CF object using an instantiated factorizer. */ @@ -165,43 +184,6 @@ void CF::Train(const arma::sp_mat& data, Timer::Stop("cf_factorization"); } -//! Serialize the model. -template -template -void CF::serialize(Archive& ar, const unsigned int /* version */) -{ - // This model is simple; just serialize all the members. No special handling - // required. - ar & BOOST_SERIALIZATION_NVP(numUsersForSimilarity); - ar & BOOST_SERIALIZATION_NVP(rank); - ar & BOOST_SERIALIZATION_NVP(w); - ar & BOOST_SERIALIZATION_NVP(h); - ar & BOOST_SERIALIZATION_NVP(cleanedData); - ar & BOOST_SERIALIZATION_NVP(normalization); -} - - - - -// Default CF constructor. -template -CF::CF(const size_t numUsersForSimilarity, - const size_t rank, - const NormalizationType normalization) : - numUsersForSimilarity(numUsersForSimilarity), - rank(rank), - normalization(normalization) -{ - // Validate neighbourhood size. - if (numUsersForSimilarity < 1) - { - Log::Warn << "CF::CF(): neighbourhood size should be > 0 (" - << numUsersForSimilarity << " given). Setting value to 5.\n"; - // Set default value of 5. - this->numUsersForSimilarity = 5; - } -} - template void CF::GetRecommendations(const size_t numRecs, arma::Mat& recommendations) @@ -441,8 +423,20 @@ void CF::CleanData(const arma::mat& data, arma::sp_mat& clean cleanedData = arma::sp_mat(locations, values, maxItemID, maxUserID); } - - +//! Serialize the model. +template +template +void CF::serialize(Archive& ar, const unsigned int /* version */) +{ + // This model is simple; just serialize all the members. No special handling + // required. + ar & BOOST_SERIALIZATION_NVP(numUsersForSimilarity); + ar & BOOST_SERIALIZATION_NVP(rank); + ar & BOOST_SERIALIZATION_NVP(w); + ar & BOOST_SERIALIZATION_NVP(h); + ar & BOOST_SERIALIZATION_NVP(cleanedData); + ar & BOOST_SERIALIZATION_NVP(normalization); +} } // namespace cf } // namespace mlpack diff --git a/src/mlpack/methods/cf/normalization/no_normalization.hpp b/src/mlpack/methods/cf/normalization/no_normalization.hpp index 30bb09dad2..27680fa251 100644 --- a/src/mlpack/methods/cf/normalization/no_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/no_normalization.hpp @@ -60,7 +60,7 @@ class NoNormalization * Do nothing. * * @param combinations User/Item combinations. - * @param predictions Predicted Ratings for each User/Item combination. + * @param predictions Predicted ratings for each user/item combination. */ inline void Denormalize(const arma::Mat& /* combinations */, const arma::vec& /* predictions */) const From 9f9a582a9689bc1ec38200ed48edac9aa96b4a6f Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Thu, 17 May 2018 23:49:06 +0800 Subject: [PATCH 09/79] style fix --- src/mlpack/methods/cf/cf_impl.hpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/cf/cf_impl.hpp b/src/mlpack/methods/cf/cf_impl.hpp index 170a905fa8..d7b4cb3337 100644 --- a/src/mlpack/methods/cf/cf_impl.hpp +++ b/src/mlpack/methods/cf/cf_impl.hpp @@ -129,7 +129,8 @@ CF::CF(const arma::sp_mat& data, template template -void CF::Train(const arma::mat& data, FactorizerType factorizer) +void CF::Train(const arma::mat& data, + FactorizerType factorizer) { CleanData(data, cleanedData); @@ -297,7 +298,8 @@ void CF::GetRecommendations(const size_t numRecs, // Predict the rating for a single user/item combination. template -double CF::Predict(const size_t user, const size_t item) const +double CF::Predict(const size_t user, + const size_t item) const { // First, we need to find the nearest neighbors of the given user. // We'll use the same technique as for GetRecommendations(). @@ -398,7 +400,8 @@ void CF::Predict(const arma::Mat& combinations, } template -void CF::CleanData(const arma::mat& data, arma::sp_mat& cleanedData) +void CF::CleanData(const arma::mat& data, + arma::sp_mat& cleanedData) { // Generate list of locations for batch insert constructor for sparse // matrices. @@ -426,7 +429,8 @@ void CF::CleanData(const arma::mat& data, arma::sp_mat& clean //! Serialize the model. template template -void CF::serialize(Archive& ar, const unsigned int /* version */) +void CF::serialize(Archive& ar, + const unsigned int /* version */) { // This model is simple; just serialize all the members. No special handling // required. From 5ac5cc8a3a47a40a977e14a5239726d85e1818c1 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Sat, 19 May 2018 23:26:21 +0800 Subject: [PATCH 10/79] bug fix --- src/mlpack/methods/cf/cf.hpp | 10 +++------- src/mlpack/methods/cf/cf_impl.hpp | 28 +++++++++++----------------- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/src/mlpack/methods/cf/cf.hpp b/src/mlpack/methods/cf/cf.hpp index 5691ec974c..cef4776e06 100644 --- a/src/mlpack/methods/cf/cf.hpp +++ b/src/mlpack/methods/cf/cf.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -92,8 +93,7 @@ class CF * call Train() before calling GetRecommendations() or any other functions! */ CF(const size_t numUsersForSimilarity = 5, - const size_t rank = 0, - const NormalizationType normalization = NormalizationType()); + const size_t rank = 0); /** * Initialize the CF object using an instantiated factorizer, immediately @@ -110,14 +110,12 @@ class CF * @param factorizer Instantiated factorizer object. * @param numUsersForSimilarity Size of the neighborhood. * @param rank Rank parameter for matrix factorization. - * @param normalization Instantiated normalization object. */ template CF(const arma::mat& data, FactorizerType factorizer = FactorizerType(), const size_t numUsersForSimilarity = 5, - const size_t rank = 0, - const NormalizationType normalization = NormalizationType()); + const size_t rank = 0); /** * Initialize the CF object using an instantiated factorizer, immediately @@ -136,14 +134,12 @@ class CF * @param factorizer Instantiated factorizer object. * @param numUsersForSimilarity Size of the neighborhood. * @param rank Rank parameter for matrix factorization. - * @param normalization Instantiated normalization object. */ template CF(const arma::sp_mat& data, FactorizerType factorizer = FactorizerType(), const size_t numUsersForSimilarity = 5, const size_t rank = 0, - const NormalizationType normalization = NormalizationType(), const typename std::enable_if_t< !FactorizerTraits::UsesCoordinateList>* = 0); diff --git a/src/mlpack/methods/cf/cf_impl.hpp b/src/mlpack/methods/cf/cf_impl.hpp index d7b4cb3337..eff281ae00 100644 --- a/src/mlpack/methods/cf/cf_impl.hpp +++ b/src/mlpack/methods/cf/cf_impl.hpp @@ -53,11 +53,9 @@ void ApplyFactorizer(FactorizerType& factorizer, // Default CF constructor. template CF::CF(const size_t numUsersForSimilarity, - const size_t rank, - const NormalizationType normalization) : + const size_t rank) : numUsersForSimilarity(numUsersForSimilarity), - rank(rank), - normalization(normalization) + rank(rank) { // Validate neighbourhood size. if (numUsersForSimilarity < 1) @@ -77,11 +75,9 @@ template CF::CF(const arma::mat& data, FactorizerType factorizer, const size_t numUsersForSimilarity, - const size_t rank, - const NormalizationType normalization) : + const size_t rank) : numUsersForSimilarity(numUsersForSimilarity), - rank(rank), - normalization(normalization) + rank(rank) { // Validate neighbourhood size. if (numUsersForSimilarity < 1) @@ -92,8 +88,6 @@ CF::CF(const arma::mat& data, this->numUsersForSimilarity = 5; } - // Normalize data and train. - normalization.Normalize(data); Train(data, factorizer); } @@ -106,12 +100,10 @@ CF::CF(const arma::sp_mat& data, FactorizerType factorizer, const size_t numUsersForSimilarity, const size_t rank, - const NormalizationType normalization, const typename std::enable_if_t< !FactorizerTraits::UsesCoordinateList>*) : numUsersForSimilarity(numUsersForSimilarity), - rank(rank), - normalization(normalization) + rank(rank) { // Validate neighbourhood size. if (numUsersForSimilarity < 1) @@ -122,8 +114,6 @@ CF::CF(const arma::sp_mat& data, this->numUsersForSimilarity = 5; } - // Normalize data and train. - normalization.Normalize(data); Train(data, factorizer); } @@ -132,7 +122,10 @@ template void CF::Train(const arma::mat& data, FactorizerType factorizer) { - CleanData(data, cleanedData); + // Make a copy of data before performing normalization. + arma::mat ds(data); + normalization.Normalize(ds); + CleanData(ds, cleanedData); // Check if the user wanted us to choose a rank for them. if (rank == 0) @@ -152,7 +145,7 @@ void CF::Train(const arma::mat& data, // Decompose the data matrix (which is in coordinate list form) to user and // data matrices. Timer::Start("cf_factorization"); - ApplyFactorizer(factorizer, data, cleanedData, this->rank, w, h); + ApplyFactorizer(factorizer, ds, cleanedData, this->rank, w, h); Timer::Stop("cf_factorization"); } @@ -164,6 +157,7 @@ void CF::Train(const arma::sp_mat& data, FactorizerType>::UsesCoordinateList>*) { cleanedData = data; + normalization.Normalize(cleanedData); // Check if the user wanted us to choose a rank for them. if (rank == 0) From 5a944343c387ebb27c4db4b96553c054f7789db4 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Sat, 19 May 2018 23:28:22 +0800 Subject: [PATCH 11/79] add overall mean normalization --- .../cf/normalization/no_normalization.hpp | 6 +- .../overall_mean_normalization.hpp | 108 ++++++++++++++++++ 2 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp diff --git a/src/mlpack/methods/cf/normalization/no_normalization.hpp b/src/mlpack/methods/cf/normalization/no_normalization.hpp index 27680fa251..0e7c970716 100644 --- a/src/mlpack/methods/cf/normalization/no_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/no_normalization.hpp @@ -50,8 +50,8 @@ class NoNormalization * @param rating Computed rating before denormalization. */ inline double Denormalize(const int /* user */, - const int /* item */, - const double rating) const + const int /* item */, + const double rating) const { return rating; } @@ -63,7 +63,7 @@ class NoNormalization * @param predictions Predicted ratings for each user/item combination. */ inline void Denormalize(const arma::Mat& /* combinations */, - const arma::vec& /* predictions */) const + const arma::vec& /* predictions */) const { } /** diff --git a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp new file mode 100644 index 0000000000..d911cb2bb1 --- /dev/null +++ b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp @@ -0,0 +1,108 @@ +/** + * @file overall_mean_normalization.hpp + * @author Wenhao Huang + * + * This class performs overall mean normalization on raw ratings. In another + * word, this class is used to remove global effect of overall mean. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_CF_NORMALIZATION_OVERALL_MEAN_NORMALIZATION_HPP +#define MLPACK_METHODS_CF_NORMALIZATION_OVERALL_MEAN_NORMALIZATION_HPP + +#include + +namespace mlpack { +namespace cf { + +/** + * This normalization class performs overall mean normalization on raw ratings. + */ +class OverallMeanNormalization +{ + public: + // Empty constructor. + OverallMeanNormalization() { } + + /** + * Normalize the data by subtracting the mean of all existing ratings. + * + * @param data Input dataset in the form of coordinate list. + */ + void Normalize(arma::mat& data) + { + mean = arma::mean(data.row(2)); + data.row(2) -= mean; + } + + /** + * Normalize the data by subtracting the mean of all existing ratings. + * + * @param cleanedData Sparse matrix data. + */ + void Normalize(arma::sp_mat& cleanedData) + { + // Caculate mean of all non zero ratings. + mean = arma::accu(cleanedData) / cleanedData.n_nonzero; + // Subtract mean from all non zero ratings. + arma::sp_mat::iterator it = cleanedData.begin(); + arma::sp_mat::iterator it_end = cleanedData.end(); + for (; it != it_end; it++) + *it = *it - mean; + } + + /** + * Denormalize computed rating by adding mean. + * + * @param user User ID. + * @param item Item ID. + * @param rating Computed rating before denormalization. + */ + double Denormalize(const int /* user */, + const int /* item */, + const double rating) const + { + return rating + mean; + } + + /** + * Denormalize computed rating by adding mean. + * + * @param combinations User/Item combinations. + * @param predictions Predicted ratings for each user/item combination. + */ + void Denormalize(const arma::Mat& /* combinations */, + arma::vec& predictions) const + { + predictions += mean; + } + + /** + * Return mean. + */ + double Mean() const + { + return mean; + } + + /** + * Serialization. + */ + template + void serialize(Archive& ar, const unsigned int /* version */) + { + ar & BOOST_SERIALIZATION_NVP(mean); + } + + private: + //! Mean of all existing ratings. + double mean; +}; + +} // namespace cf +} // namespace mlpack + +#endif From 2cffbb3840204db813e5351d8e2025e56523771e Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Sat, 19 May 2018 23:38:21 +0800 Subject: [PATCH 12/79] add user/item normalization --- .../methods/cf/normalization/CMakeLists.txt | 3 + .../normalization/item_mean_normalization.hpp | 132 ++++++++++++++++++ .../normalization/user_mean_normalization.hpp | 132 ++++++++++++++++++ 3 files changed, 267 insertions(+) create mode 100644 src/mlpack/methods/cf/normalization/item_mean_normalization.hpp create mode 100644 src/mlpack/methods/cf/normalization/user_mean_normalization.hpp diff --git a/src/mlpack/methods/cf/normalization/CMakeLists.txt b/src/mlpack/methods/cf/normalization/CMakeLists.txt index aaf2fb4c5e..ecf0f47ecc 100644 --- a/src/mlpack/methods/cf/normalization/CMakeLists.txt +++ b/src/mlpack/methods/cf/normalization/CMakeLists.txt @@ -2,6 +2,9 @@ # Anything not in this list will not be compiled into mlpack. set(SOURCES no_normalization.hpp + overall_mean_normalization.hpp + user_mean_normalization.hpp + item_mean_normalization.hpp ) # Add directory name to sources. diff --git a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp new file mode 100644 index 0000000000..0c986cbb83 --- /dev/null +++ b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp @@ -0,0 +1,132 @@ +/** + * @file item_mean_normalization.hpp + * @author Wenhao Huang + * + * This class performs item mean normalization on raw ratings. In another + * word, this class is used to remove global effect of item mean. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_CF_NORMALIZATION_ITEM_MEAN_NORMALIZATION_HPP +#define MLPACK_METHODS_CF_NORMALIZATION_ITEM_MEAN_NORMALIZATION_HPP + +#include + +namespace mlpack { +namespace cf { + +/** + * This normalization class performs item mean normalization on raw ratings. + */ +class ItemMeanNormalization +{ + public: + // Empty constructor. + ItemMeanNormalization() { } + + /** + * Normalize the data by subtracting item mean from each of existing rating. + * + * @param data Input dataset in the form of coordinate list. + */ + void Normalize(arma::mat& data) + { + const size_t itemNum = arma::max(data.row(1)) + 1; + // Should we use overall mean if an item has no rating? + itemMean = arma::vec(itemNum, arma::fill:zeros); + // Number of ratings for each item. + arma::vec ratingNum(itemNum, arma::fill::zeros); + + // Sum ratings for each item. + data.each_col([&](vec& datapoint) { + const size_t item = (size_t) datapoint(1); + const double rating = datapoint(2); + itemMean(item) += rating; + ratingNum(item) += 1; + }); + + // Calculate item mean and subtract item mean from ratings. + // Set item mean to 0 if the item has no rating. + for (int i = 0; i < itemNum; i++) + if ((size_t) itemNum != 0) + itemMean(i) /= itemNum(i); + data.each_col([&](vec& datapoint) { + const size_t item = (size_t) datapoint(1); + datapoint(2) -= itemMean(item); + }); + } + + /** + * Normalize the data by subtracting item mean from each of existing rating. + * + * @param cleanedData Sparse matrix data. + */ + void Normalize(arma::sp_mat& cleanedData) + { + itemMean = arma::mean(cleanedData, 1); + + arma::sp_mat::iterator it = cleanedData.begin(); + arma::sp_mat::iterator it_end = cleanedData.end(); + for (; it != it_end; it++) + *it = *it - itemMean(it.row()); + } + + /** + * Denormalize computed rating by adding item mean. + * + * @param user User ID. + * @param item Item ID. + * @param rating Computed rating before denormalization. + */ + double Denormalize(const int /* user */, + const int item, + const double rating) const + { + return rating + itemMean(item); + } + + /** + * Denormalize computed rating by adding item mean. + * + * @param combinations User/Item combinations. + * @param predictions Predicted ratings for each user/item combination. + */ + void Denormalize(const arma::Mat& combinations, + arma::vec& predictions) const + { + for (int i = 0; i < predictions.n_elem; i++) + { + const size_t item = combinations(1, i); + predictions(i) += itemMean(item); + } + } + + /** + * Return item mean. + */ + arma::vec ItemMean() const + { + return itemMean; + } + + /** + * Serialization. + */ + template + void serialize(Archive& ar, const unsigned int /* version */) + { + ar & BOOST_SERIALIZATION_NVP(itemMean); + } + + private: + //! item mean. + arma::vec itemMean; +}; + +} // namespace cf +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp new file mode 100644 index 0000000000..dd0878443a --- /dev/null +++ b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp @@ -0,0 +1,132 @@ +/** + * @file user_mean_normalization.hpp + * @author Wenhao Huang + * + * This class performs user mean normalization on raw ratings. In another + * word, this class is used to remove global effect of user mean. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_CF_NORMALIZATION_USER_MEAN_NORMALIZATION_HPP +#define MLPACK_METHODS_CF_NORMALIZATION_USER_MEAN_NORMALIZATION_HPP + +#include + +namespace mlpack { +namespace cf { + +/** + * This normalization class performs user mean normalization on raw ratings. + */ +class UserMeanNormalization +{ + public: + // Empty constructor. + UserMeanNormalization() { } + + /** + * Normalize the data by subtracting user mean from each of existing rating. + * + * @param data Input dataset in the form of coordinate list. + */ + void Normalize(arma::mat& data) + { + const size_t userNum = arma::max(data.row(0)) + 1; + // Should we use overall mean if a user has no rating? + userMean = arma::vec(userNum, arma::fill:zeros); + // Number of ratings for each user. + arma::vec ratingNum(userNum, arma::fill::zeros); + + // Sum ratings for each user. + data.each_col([&](vec& datapoint) { + const size_t user = (size_t) datapoint(0); + const double rating = datapoint(2); + userMean(user) += rating; + ratingNum(user) += 1; + }); + + // Calculate user mean and subtract user mean from ratings. + // Set user mean to 0 if the user has no rating. + for (int i = 0; i < userNum; i++) + if ((size_t) ratingNum(i) != 0) + userMean(i) /= ratingNum(i); + data.each_col([&](vec& datapoint) { + const size_t user = (size_t) datapoint(0); + datapoint(2) -= userMean(user); + }); + } + + /** + * Normalize the data by subtracting user mean from each of existing rating. + * + * @param cleanedData Sparse matrix data. + */ + void Normalize(arma::sp_mat& cleanedData) + { + userMean = arma::mean(cleanedData, 0); + + arma::sp_mat::iterator it = cleanedData.begin(); + arma::sp_mat::iterator it_end = cleanedData.end(); + for (; it != it_end; it++) + *it = *it - userMean(it.col()); + } + + /** + * Denormalize computed rating by adding user mean. + * + * @param user User ID. + * @param item Item ID. + * @param rating Computed rating before denormalization. + */ + double Denormalize(const int user, + const int /* item */, + const double rating) const + { + return rating + userMean(user); + } + + /** + * Denormalize computed rating by adding user mean. + * + * @param combinations User/Item combinations. + * @param predictions Predicted ratings for each user/item combination. + */ + void Denormalize(const arma::Mat& combinations, + arma::vec& predictions) const + { + for (int i = 0; i < predictions.n_elem; i++) + { + const size_t user = combinations(0, i); + predictions(i) += userMean(user); + } + } + + /** + * Return user mean. + */ + arma::vec UserMean() const + { + return userMean; + } + + /** + * Serialization. + */ + template + void serialize(Archive& ar, const unsigned int /* version */) + { + ar & BOOST_SERIALIZATION_NVP(userMean); + } + + private: + //! User mean. + arma::vec userMean; +}; + +} // namespace cf +} // namespace mlpack + +#endif From 357b6ca192c3c317b19477bd1567fef334f8f399 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Sun, 20 May 2018 00:17:23 +0800 Subject: [PATCH 13/79] bugfix --- src/mlpack/methods/cf/cf.hpp | 2 ++ .../cf/normalization/item_mean_normalization.hpp | 14 +++++++------- .../normalization/overall_mean_normalization.hpp | 2 +- .../cf/normalization/user_mean_normalization.hpp | 12 ++++++------ 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/mlpack/methods/cf/cf.hpp b/src/mlpack/methods/cf/cf.hpp index cef4776e06..d75a4dc4e2 100644 --- a/src/mlpack/methods/cf/cf.hpp +++ b/src/mlpack/methods/cf/cf.hpp @@ -23,6 +23,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp index 0c986cbb83..92e2838b98 100644 --- a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp @@ -36,12 +36,12 @@ class ItemMeanNormalization { const size_t itemNum = arma::max(data.row(1)) + 1; // Should we use overall mean if an item has no rating? - itemMean = arma::vec(itemNum, arma::fill:zeros); + itemMean = arma::vec(itemNum, arma::fill::zeros); // Number of ratings for each item. arma::vec ratingNum(itemNum, arma::fill::zeros); // Sum ratings for each item. - data.each_col([&](vec& datapoint) { + data.each_col([&](arma::vec& datapoint) { const size_t item = (size_t) datapoint(1); const double rating = datapoint(2); itemMean(item) += rating; @@ -50,10 +50,10 @@ class ItemMeanNormalization // Calculate item mean and subtract item mean from ratings. // Set item mean to 0 if the item has no rating. - for (int i = 0; i < itemNum; i++) + for (size_t i = 0; i < itemNum; i++) if ((size_t) itemNum != 0) - itemMean(i) /= itemNum(i); - data.each_col([&](vec& datapoint) { + itemMean(i) /= ratingNum(i); + data.each_col([&](arma::vec& datapoint) { const size_t item = (size_t) datapoint(1); datapoint(2) -= itemMean(item); }); @@ -69,7 +69,7 @@ class ItemMeanNormalization itemMean = arma::mean(cleanedData, 1); arma::sp_mat::iterator it = cleanedData.begin(); - arma::sp_mat::iterator it_end = cleanedData.end(); + arma::sp_mat::iterator it_end = cleanedData.end(); for (; it != it_end; it++) *it = *it - itemMean(it.row()); } @@ -97,7 +97,7 @@ class ItemMeanNormalization void Denormalize(const arma::Mat& combinations, arma::vec& predictions) const { - for (int i = 0; i < predictions.n_elem; i++) + for (size_t i = 0; i < predictions.n_elem; i++) { const size_t item = combinations(1, i); predictions(i) += itemMean(item); diff --git a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp index d911cb2bb1..7c652e89ff 100644 --- a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp @@ -49,7 +49,7 @@ class OverallMeanNormalization mean = arma::accu(cleanedData) / cleanedData.n_nonzero; // Subtract mean from all non zero ratings. arma::sp_mat::iterator it = cleanedData.begin(); - arma::sp_mat::iterator it_end = cleanedData.end(); + arma::sp_mat::iterator it_end = cleanedData.end(); for (; it != it_end; it++) *it = *it - mean; } diff --git a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp index dd0878443a..f9eda8445b 100644 --- a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp @@ -36,12 +36,12 @@ class UserMeanNormalization { const size_t userNum = arma::max(data.row(0)) + 1; // Should we use overall mean if a user has no rating? - userMean = arma::vec(userNum, arma::fill:zeros); + userMean = arma::vec(userNum, arma::fill::zeros); // Number of ratings for each user. arma::vec ratingNum(userNum, arma::fill::zeros); // Sum ratings for each user. - data.each_col([&](vec& datapoint) { + data.each_col([&](arma::vec& datapoint) { const size_t user = (size_t) datapoint(0); const double rating = datapoint(2); userMean(user) += rating; @@ -50,10 +50,10 @@ class UserMeanNormalization // Calculate user mean and subtract user mean from ratings. // Set user mean to 0 if the user has no rating. - for (int i = 0; i < userNum; i++) + for (size_t i = 0; i < userNum; i++) if ((size_t) ratingNum(i) != 0) userMean(i) /= ratingNum(i); - data.each_col([&](vec& datapoint) { + data.each_col([&](arma::vec& datapoint) { const size_t user = (size_t) datapoint(0); datapoint(2) -= userMean(user); }); @@ -69,7 +69,7 @@ class UserMeanNormalization userMean = arma::mean(cleanedData, 0); arma::sp_mat::iterator it = cleanedData.begin(); - arma::sp_mat::iterator it_end = cleanedData.end(); + arma::sp_mat::iterator it_end = cleanedData.end(); for (; it != it_end; it++) *it = *it - userMean(it.col()); } @@ -97,7 +97,7 @@ class UserMeanNormalization void Denormalize(const arma::Mat& combinations, arma::vec& predictions) const { - for (int i = 0; i < predictions.n_elem; i++) + for (size_t i = 0; i < predictions.n_elem; i++) { const size_t user = combinations(0, i); predictions(i) += userMean(user); From 457dcaf4cf1cdfc9605f9cdc56eaa84859d26738 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Sun, 20 May 2018 14:32:45 +0800 Subject: [PATCH 14/79] add z-score normalization --- .../normalization/z_score_normalization.hpp | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 src/mlpack/methods/cf/normalization/z_score_normalization.hpp diff --git a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp new file mode 100644 index 0000000000..0cc8bcd5c4 --- /dev/null +++ b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp @@ -0,0 +1,122 @@ +/** + * @file z_score_normalization.hpp + * @author Wenhao Huang + * + * This class performs z-score normalization on raw ratings. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_CF_NORMALIZATION_Z_SCORE_NORMALIZATION_HPP +#define MLPACK_METHODS_CF_NORMALIZATION_Z_SCORE_NORMALIZATION_HPP + +#include + +namespace mlpack { +namespace cf { + +/** + * This normalization class performs z-score normalization on raw ratings. + */ +class ZScoreNormalization +{ + public: + // Empty constructor. + ZScoreNormalization() { } + + /** + * Normalize the data to zero mean and one standard deviation. + * + * @param data Input dataset in the form of coordinate list. + */ + void Normalize(arma::mat& data) + { + mean = arma::mean(data.row(2)); + stddev = arma::stddev(data.row(2)); + data.row(2) = (data.row(2) - mean) / stddev; + } + + /** + * Normalize the data to zero mean and one standard deviation. + * + * @param cleanedData Sparse matrix data. + */ + void Normalize(arma::sp_mat& cleanedData) + { + arma::vec ratings = arma::nonzeros(cleanedData); + // Caculate mean and stdev of all non zero ratings. + mean = arma::mean(ratings); + stddev = arma::stddev(ratings); + + // Subtract mean from existing rating and divide it by stddev. + arma::sp_mat::iterator it = cleanedData.begin(); + arma::sp_mat::iterator it_end = cleanedData.end(); + for (; it != it_end; it++) + *it = (*it - mean) / stddev; + } + + /** + * Denormalize computed rating by adding mean and multiplying stddev. + * + * @param user User ID. + * @param item Item ID. + * @param rating Computed rating before denormalization. + */ + double Denormalize(const int /* user */, + const int /* item */, + const double rating) const + { + return (rating + mean) * stddev; + } + + /** + * Denormalize computed rating by adding mean and multiplying stddev. + * + * @param combinations User/Item combinations. + * @param predictions Predicted ratings for each user/item combination. + */ + void Denormalize(const arma::Mat& /* combinations */, + arma::vec& predictions) const + { + predictions = (predictions + mean) * stddev; + } + + /** + * Return mean. + */ + double Mean() const + { + return mean; + } + + /** + * Return stddev. + */ + double Stddev() const + { + return stddev; + } + + /** + * Serialization. + */ + template + void serialize(Archive& ar, const unsigned int /* version */) + { + ar & BOOST_SERIALIZATION_NVP(mean); + ar & BOOST_SERIALIZATION_NVP(stddev); + } + + private: + //! Mean of all existing ratings. + double mean; + //! Standard deviation of all existing ratings. + double stddev; +}; + +} // namespace cf +} // namespace mlpack + +#endif From 2a0f2cd46487745ea5923f24347d66a001d299b4 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Sun, 20 May 2018 18:12:12 +0800 Subject: [PATCH 15/79] add combined normalization --- src/mlpack/methods/cf/cf.hpp | 3 - .../methods/cf/normalization/CMakeLists.txt | 2 + .../normalization/combined_normalization.hpp | 181 ++++++++++++++++++ .../normalization/item_mean_normalization.hpp | 2 +- 4 files changed, 184 insertions(+), 4 deletions(-) create mode 100644 src/mlpack/methods/cf/normalization/combined_normalization.hpp diff --git a/src/mlpack/methods/cf/cf.hpp b/src/mlpack/methods/cf/cf.hpp index d75a4dc4e2..3974a95e42 100644 --- a/src/mlpack/methods/cf/cf.hpp +++ b/src/mlpack/methods/cf/cf.hpp @@ -22,9 +22,6 @@ #include #include #include -#include -#include -#include #include #include #include diff --git a/src/mlpack/methods/cf/normalization/CMakeLists.txt b/src/mlpack/methods/cf/normalization/CMakeLists.txt index ecf0f47ecc..648bf7eca5 100644 --- a/src/mlpack/methods/cf/normalization/CMakeLists.txt +++ b/src/mlpack/methods/cf/normalization/CMakeLists.txt @@ -5,6 +5,8 @@ set(SOURCES overall_mean_normalization.hpp user_mean_normalization.hpp item_mean_normalization.hpp + z_score_normalization.hpp + combined_normalization.hpp ) # Add directory name to sources. diff --git a/src/mlpack/methods/cf/normalization/combined_normalization.hpp b/src/mlpack/methods/cf/normalization/combined_normalization.hpp new file mode 100644 index 0000000000..8cc9a4135d --- /dev/null +++ b/src/mlpack/methods/cf/normalization/combined_normalization.hpp @@ -0,0 +1,181 @@ +/** + * @file combined_normalization.hpp + * @author Wenhao Huang + * + * CombinedNormalization is a class template for performing a sequence of data + * normalization methods which are specified by template parameters. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_CF_NORMALIZATION_COMBINED_NORMALIZATION_HPP +#define MLPACK_METHODS_CF_NORMALIZATION_COMBINED_NORMALIZATION_HPP + +#include + +namespace mlpack { +namespace cf { + +/** + * This normalization class performs a sequence of normalization methods on + * raw ratings. + */ +template +class CombinedNormalization +{ + public: + using TupleType = std::tuple; + + // Empty constructor. + CombinedNormalization() { } + + /** + * Normalize the data. + * + * @param data Input dataset in the form of coordinate list. + */ + void Normalize(arma::mat& data) + { + SequenceNormalize<0>(data); + } + + /** + * Normalize the data by subtracting the mean of all existing ratings. + * + * @param cleanedData Sparse matrix data. + */ + void Normalize(arma::sp_mat& cleanedData) + { + SequenceNormalize<0>(cleanedData); + } + + /** + * Denormalize computed rating by adding mean. + * + * @param user User ID. + * @param item Item ID. + * @param rating Computed rating before denormalization. + */ + double Denormalize(const int user, + const int item, + const double rating) const + { + return SequenceDenormalize<0>(user, item, rating); + } + + /** + * Denormalize computed rating by adding mean. + * + * @param combinations User/Item combinations. + * @param predictions Predicted ratings for each user/item combination. + */ + void Denormalize(const arma::Mat& combinations, + arma::vec& predictions) const + { + SequenceDenormalize<0>(combinations, predictions); + } + + /** + * Return normalizations. + */ + TupleType Normalizations() const + { + return normalizations; + } + + /** + * Serialization. + */ + template + void serialize(Archive& ar, const unsigned int /* version */) + { + // Boost does not support tuple serialization??? + ar & BOOST_SERIALIZATION_NVP(normalizations); + } + + private: + + template< + int I, /* Which normalization in tuple to use */ + typename = std::enable_if_t<(I < std::tuple_size::value)>> + void SequenceNormalize(arma::mat& data) + { + std::get(normalizations).Normalize(data); + SequenceNormalize(data); + } + + template< + int I, /* Which normalization in tuple to use */ + typename = std::enable_if_t<(I >= std::tuple_size::value)>, + typename = void> + void SequenceNormalize(arma::mat& /* data */) { } + + template< + int I, /* Which normalization in tuple to use */ + typename = std::enable_if_t<(I < std::tuple_size::value)>> + void SequenceNormalize(arma::sp_mat& cleanedData) + { + std::get(normalizations).Normalize(cleanedData); + SequenceNormalize(cleanedData); + } + + template< + int I, /* Which normalization in tuple to use */ + typename = std::enable_if_t<(I >= std::tuple_size::value)>, + typename = void> + void SequenceNormalize(arma::sp_mat& /* cleanedData */) { } + + template< + int I, /* Which normalization in tuple to use */ + typename = std::enable_if_t<(I < std::tuple_size::value)>> + double SequenceDenormalize(const int user, + const int item, + const double rating) const + { + // The order of denormalization should be the reversed order + // of normalization. + double realRating = SequenceDenormalize(user, item, rating); + realRating = std::get(normalizations).Denormalize(user, item, realRating); + return realRating; + } + + template< + int I, /* Which normalization in tuple to use */ + typename = std::enable_if_t<(I >= std::tuple_size::value)>, + typename = void> + double SequenceDenormalize(const int /* user */, + const int /* item */, + const double rating) const + { + return rating; + } + + template< + int I, /* Which normalization in tuple to use */ + typename = std::enable_if_t<(I < std::tuple_size::value)>> + void SequenceDenormalize(const arma::Mat& combinations, + arma::vec& predictions) const + { + // The order of denormalization should be the reversed order + // of normalization. + SequenceDenormalize(combinations, predictions); + std::get(normalizations).Denormalize(combinations, predictions); + } + + template< + int I, /* Which normalization in tuple to use */ + typename = std::enable_if_t<(I >= std::tuple_size::value)>, + typename = void> + void SequenceDenormalize(const arma::Mat& /* combinations */, + arma::vec& /* predictions */) const { } + + //! A tuple of all normalizations. + TupleType normalizations; +}; + +} // namespace cf +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp index 92e2838b98..14f00e070e 100644 --- a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp @@ -51,7 +51,7 @@ class ItemMeanNormalization // Calculate item mean and subtract item mean from ratings. // Set item mean to 0 if the item has no rating. for (size_t i = 0; i < itemNum; i++) - if ((size_t) itemNum != 0) + if ((size_t) ratingNum(i) != 0) itemMean(i) /= ratingNum(i); data.each_col([&](arma::vec& datapoint) { const size_t item = (size_t) datapoint(1); From 3ccc4de65565af225daa49483e5b73a20b901a6a Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Sun, 20 May 2018 22:08:58 +0800 Subject: [PATCH 16/79] update comments --- .../normalization/combined_normalization.hpp | 38 ++++++++++++------- .../normalization/item_mean_normalization.hpp | 6 +-- .../normalization/user_mean_normalization.hpp | 4 +- .../normalization/z_score_normalization.hpp | 2 +- 4 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/mlpack/methods/cf/normalization/combined_normalization.hpp b/src/mlpack/methods/cf/normalization/combined_normalization.hpp index 8cc9a4135d..4c3522c1a3 100644 --- a/src/mlpack/methods/cf/normalization/combined_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/combined_normalization.hpp @@ -3,7 +3,7 @@ * @author Wenhao Huang * * CombinedNormalization is a class template for performing a sequence of data - * normalization methods which are specified by template parameters. + * normalization methods which are specified by template parameter. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -32,7 +32,7 @@ class CombinedNormalization CombinedNormalization() { } /** - * Normalize the data. + * Normalize the data by calling Normalize() in each normalization object. * * @param data Input dataset in the form of coordinate list. */ @@ -42,7 +42,7 @@ class CombinedNormalization } /** - * Normalize the data by subtracting the mean of all existing ratings. + * Normalize the data by calling Normalize() in each normalization object. * * @param cleanedData Sparse matrix data. */ @@ -52,7 +52,9 @@ class CombinedNormalization } /** - * Denormalize computed rating by adding mean. + * Denormalize rating by calling Denormalize() in each normalization object. + * Note that the order of objects calling Denormalize() should be the + * reversed order of objects calling Normalize(). * * @param user User ID. * @param item Item ID. @@ -66,7 +68,9 @@ class CombinedNormalization } /** - * Denormalize computed rating by adding mean. + * Denormalize rating by calling Denormalize() in each normalization object. + * Note that the order of objects calling Denormalize() should be the + * reversed order of objects calling Normalize(). * * @param combinations User/Item combinations. * @param predictions Predicted ratings for each user/item combination. @@ -78,7 +82,7 @@ class CombinedNormalization } /** - * Return normalizations. + * Return normalizations tuple. */ TupleType Normalizations() const { @@ -96,7 +100,10 @@ class CombinedNormalization } private: + //! A tuple of all normalization objects. + TupleType normalizations; + //! Unpack normalizations tuple to normalize data. template< int I, /* Which normalization in tuple to use */ typename = std::enable_if_t<(I < std::tuple_size::value)>> @@ -105,13 +112,15 @@ class CombinedNormalization std::get(normalizations).Normalize(data); SequenceNormalize(data); } - + + //! End of tuple unpacking. template< int I, /* Which normalization in tuple to use */ typename = std::enable_if_t<(I >= std::tuple_size::value)>, typename = void> void SequenceNormalize(arma::mat& /* data */) { } + //! Unpack normalizations tuple to normalize cleanedData. template< int I, /* Which normalization in tuple to use */ typename = std::enable_if_t<(I < std::tuple_size::value)>> @@ -121,12 +130,14 @@ class CombinedNormalization SequenceNormalize(cleanedData); } + //! End of tuple unpacking. template< int I, /* Which normalization in tuple to use */ typename = std::enable_if_t<(I >= std::tuple_size::value)>, typename = void> void SequenceNormalize(arma::sp_mat& /* cleanedData */) { } + //! Unpack normalizations tuple to denormalize. template< int I, /* Which normalization in tuple to use */ typename = std::enable_if_t<(I < std::tuple_size::value)>> @@ -134,13 +145,15 @@ class CombinedNormalization const int item, const double rating) const { - // The order of denormalization should be the reversed order + // The order of denormalization should be the reversed order // of normalization. double realRating = SequenceDenormalize(user, item, rating); - realRating = std::get(normalizations).Denormalize(user, item, realRating); + realRating = + std::get(normalizations).Denormalize(user, item, realRating); return realRating; } + //! End of tuple unpacking. template< int I, /* Which normalization in tuple to use */ typename = std::enable_if_t<(I >= std::tuple_size::value)>, @@ -152,27 +165,26 @@ class CombinedNormalization return rating; } + //! Unpack normalizations tuple to denormalize. template< int I, /* Which normalization in tuple to use */ typename = std::enable_if_t<(I < std::tuple_size::value)>> void SequenceDenormalize(const arma::Mat& combinations, arma::vec& predictions) const { - // The order of denormalization should be the reversed order + // The order of denormalization should be the reversed order // of normalization. SequenceDenormalize(combinations, predictions); std::get(normalizations).Denormalize(combinations, predictions); } + //! End of tuple unpacking. template< int I, /* Which normalization in tuple to use */ typename = std::enable_if_t<(I >= std::tuple_size::value)>, typename = void> void SequenceDenormalize(const arma::Mat& /* combinations */, arma::vec& /* predictions */) const { } - - //! A tuple of all normalizations. - TupleType normalizations; }; } // namespace cf diff --git a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp index 14f00e070e..9bf8919625 100644 --- a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp @@ -28,14 +28,14 @@ class ItemMeanNormalization ItemMeanNormalization() { } /** - * Normalize the data by subtracting item mean from each of existing rating. + * Normalize the data by subtracting item mean from each of existing ratings. * * @param data Input dataset in the form of coordinate list. */ void Normalize(arma::mat& data) { const size_t itemNum = arma::max(data.row(1)) + 1; - // Should we use overall mean if an item has no rating? + // Should we use mean of all item means if an item has no rating? itemMean = arma::vec(itemNum, arma::fill::zeros); // Number of ratings for each item. arma::vec ratingNum(itemNum, arma::fill::zeros); @@ -60,7 +60,7 @@ class ItemMeanNormalization } /** - * Normalize the data by subtracting item mean from each of existing rating. + * Normalize the data by subtracting item mean from each of existing ratings. * * @param cleanedData Sparse matrix data. */ diff --git a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp index f9eda8445b..8ce34c970f 100644 --- a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp @@ -28,14 +28,14 @@ class UserMeanNormalization UserMeanNormalization() { } /** - * Normalize the data by subtracting user mean from each of existing rating. + * Normalize the data by subtracting user mean from each of existing ratings. * * @param data Input dataset in the form of coordinate list. */ void Normalize(arma::mat& data) { const size_t userNum = arma::max(data.row(0)) + 1; - // Should we use overall mean if a user has no rating? + // Should we use mean of all user means if a user has no rating? userMean = arma::vec(userNum, arma::fill::zeros); // Number of ratings for each user. arma::vec ratingNum(userNum, arma::fill::zeros); diff --git a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp index 0cc8bcd5c4..bfc54a8a87 100644 --- a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp @@ -45,8 +45,8 @@ class ZScoreNormalization */ void Normalize(arma::sp_mat& cleanedData) { - arma::vec ratings = arma::nonzeros(cleanedData); // Caculate mean and stdev of all non zero ratings. + arma::vec ratings = arma::nonzeros(cleanedData); mean = arma::mean(ratings); stddev = arma::stddev(ratings); From 3830674f334f9cc99df042d8d42f1847967861c8 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Sun, 20 May 2018 22:23:43 +0800 Subject: [PATCH 17/79] very small style fix --- src/mlpack/methods/cf/normalization/combined_normalization.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/cf/normalization/combined_normalization.hpp b/src/mlpack/methods/cf/normalization/combined_normalization.hpp index 4c3522c1a3..dd6b5e76a4 100644 --- a/src/mlpack/methods/cf/normalization/combined_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/combined_normalization.hpp @@ -112,7 +112,7 @@ class CombinedNormalization std::get(normalizations).Normalize(data); SequenceNormalize(data); } - + //! End of tuple unpacking. template< int I, /* Which normalization in tuple to use */ From 770c8071eb715682f5355ca78a077873000ed318 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Thu, 24 May 2018 20:33:01 +0800 Subject: [PATCH 18/79] resolve minor issues --- .../normalization/item_mean_normalization.hpp | 6 +++--- .../overall_mean_normalization.hpp | 5 ++++- .../normalization/user_mean_normalization.hpp | 6 +++--- .../normalization/z_score_normalization.hpp | 19 +++++++++++++++++-- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp index 9bf8919625..f99e1a566a 100644 --- a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp @@ -35,10 +35,9 @@ class ItemMeanNormalization void Normalize(arma::mat& data) { const size_t itemNum = arma::max(data.row(1)) + 1; - // Should we use mean of all item means if an item has no rating? itemMean = arma::vec(itemNum, arma::fill::zeros); // Number of ratings for each item. - arma::vec ratingNum(itemNum, arma::fill::zeros); + arma::Row ratingNum(itemNum, arma::fill:zeros); // Sum ratings for each item. data.each_col([&](arma::vec& datapoint) { @@ -50,8 +49,9 @@ class ItemMeanNormalization // Calculate item mean and subtract item mean from ratings. // Set item mean to 0 if the item has no rating. + // Should we use mean of all item means if an item has no rating? for (size_t i = 0; i < itemNum; i++) - if ((size_t) ratingNum(i) != 0) + if (ratingNum(i) != 0) itemMean(i) /= ratingNum(i); data.each_col([&](arma::vec& datapoint) { const size_t item = (size_t) datapoint(1); diff --git a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp index 7c652e89ff..324916df92 100644 --- a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp @@ -46,7 +46,10 @@ class OverallMeanNormalization void Normalize(arma::sp_mat& cleanedData) { // Caculate mean of all non zero ratings. - mean = arma::accu(cleanedData) / cleanedData.n_nonzero; + if (cleanedData.n_nonzero != 0) + mean = arma::accu(cleanedData) / cleanedData.n_nonzero; + else + mean = 0; // Subtract mean from all non zero ratings. arma::sp_mat::iterator it = cleanedData.begin(); arma::sp_mat::iterator it_end = cleanedData.end(); diff --git a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp index 8ce34c970f..ad01f7a139 100644 --- a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp @@ -35,10 +35,9 @@ class UserMeanNormalization void Normalize(arma::mat& data) { const size_t userNum = arma::max(data.row(0)) + 1; - // Should we use mean of all user means if a user has no rating? userMean = arma::vec(userNum, arma::fill::zeros); // Number of ratings for each user. - arma::vec ratingNum(userNum, arma::fill::zeros); + arma::Row ratingNum(userNum, arma::fill::zeros); // Sum ratings for each user. data.each_col([&](arma::vec& datapoint) { @@ -50,8 +49,9 @@ class UserMeanNormalization // Calculate user mean and subtract user mean from ratings. // Set user mean to 0 if the user has no rating. + // Should we use mean of all user means if a user has no rating? for (size_t i = 0; i < userNum; i++) - if ((size_t) ratingNum(i) != 0) + if (ratingNum(i) != 0) userMean(i) /= ratingNum(i); data.each_col([&](arma::vec& datapoint) { const size_t user = (size_t) datapoint(0); diff --git a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp index bfc54a8a87..86834c9285 100644 --- a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp @@ -35,6 +35,14 @@ class ZScoreNormalization { mean = arma::mean(data.row(2)); stddev = arma::stddev(data.row(2)); + + if (std::fabs(stddev) < 1e-14) + { + Log::Fatal << "Standard deviation of all existing ratings is 0! " + << "This may indicate that all existing ratings are the same." + << std::endl; + } + data.row(2) = (data.row(2) - mean) / stddev; } @@ -50,6 +58,13 @@ class ZScoreNormalization mean = arma::mean(ratings); stddev = arma::stddev(ratings); + if (std::fabs(stddev) < 1e-14) + { + Log::Fatal << "Standard deviation of all existing ratings is 0! " + << "This may indicate that all existing ratings are the same." + << std::endl; + } + // Subtract mean from existing rating and divide it by stddev. arma::sp_mat::iterator it = cleanedData.begin(); arma::sp_mat::iterator it_end = cleanedData.end(); @@ -68,7 +83,7 @@ class ZScoreNormalization const int /* item */, const double rating) const { - return (rating + mean) * stddev; + return rating * stddev + mean; } /** @@ -80,7 +95,7 @@ class ZScoreNormalization void Denormalize(const arma::Mat& /* combinations */, arma::vec& predictions) const { - predictions = (predictions + mean) * stddev; + predictions = predictions * stddev + mean; } /** From 0568f04b8319d0ac2d765484da82fa49c608cd1f Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Fri, 25 May 2018 23:04:45 +0800 Subject: [PATCH 19/79] style fix --- src/mlpack/methods/cf/normalization/z_score_normalization.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp index 86834c9285..2f5b6b9ab4 100644 --- a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp @@ -39,7 +39,7 @@ class ZScoreNormalization if (std::fabs(stddev) < 1e-14) { Log::Fatal << "Standard deviation of all existing ratings is 0! " - << "This may indicate that all existing ratings are the same." + << "This may indicate that all existing ratings are the same." << std::endl; } @@ -61,7 +61,7 @@ class ZScoreNormalization if (std::fabs(stddev) < 1e-14) { Log::Fatal << "Standard deviation of all existing ratings is 0! " - << "This may indicate that all existing ratings are the same." + << "This may indicate that all existing ratings are the same." << std::endl; } From d6ec5c65ecb286f721ac42fd96f724a0419a6d33 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Sat, 2 Jun 2018 23:32:12 +0800 Subject: [PATCH 20/79] bugfix --- src/mlpack/methods/cf/cf_impl.hpp | 2 +- src/mlpack/methods/cf/cf_main.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/cf/cf_impl.hpp b/src/mlpack/methods/cf/cf_impl.hpp index 5c6cdebe2a..3ae005b104 100644 --- a/src/mlpack/methods/cf/cf_impl.hpp +++ b/src/mlpack/methods/cf/cf_impl.hpp @@ -382,7 +382,7 @@ void CFType::CleanData(const arma::mat& data, //! Serialize the model. template template -void CFType::serialize(Archive& ar, const unsigned int /* version */) +void CFType::serialize(Archive& ar, const unsigned int /* version */) { // This model is simple; just serialize all the members. No special handling // required. diff --git a/src/mlpack/methods/cf/cf_main.cpp b/src/mlpack/methods/cf/cf_main.cpp index 547119ec2c..b2929b3697 100644 --- a/src/mlpack/methods/cf/cf_main.cpp +++ b/src/mlpack/methods/cf/cf_main.cpp @@ -322,7 +322,7 @@ static void mlpackMain() "test" }, true); // Load an input model. - CFType* c = std::move(CLI::GetParam*>("input_model")); + CFType<>* c = std::move(CLI::GetParam*>("input_model")); PerformAction(c); } From eb0e3e6bd9289a02a10357ccc4c149253a722d36 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Sun, 3 Jun 2018 15:39:33 +0800 Subject: [PATCH 21/79] use size_t --- src/mlpack/methods/cf/cf_impl.hpp | 27 ++++++++++--------- .../normalization/combined_normalization.hpp | 12 ++++----- .../normalization/item_mean_normalization.hpp | 4 +-- .../cf/normalization/no_normalization.hpp | 4 +-- .../overall_mean_normalization.hpp | 4 +-- .../normalization/user_mean_normalization.hpp | 4 +-- .../normalization/z_score_normalization.hpp | 4 +-- 7 files changed, 31 insertions(+), 28 deletions(-) diff --git a/src/mlpack/methods/cf/cf_impl.hpp b/src/mlpack/methods/cf/cf_impl.hpp index 3ae005b104..b0b3005803 100644 --- a/src/mlpack/methods/cf/cf_impl.hpp +++ b/src/mlpack/methods/cf/cf_impl.hpp @@ -25,9 +25,9 @@ namespace cf { // Default CF constructor. template CFType::CFType(const size_t numUsersForSimilarity, - const size_t rank) : - numUsersForSimilarity(numUsersForSimilarity), - rank(rank) + const size_t rank) : + numUsersForSimilarity(numUsersForSimilarity), + rank(rank) { // Validate neighbourhood size. if (numUsersForSimilarity < 1) @@ -139,8 +139,9 @@ void CFType::Train(const arma::sp_mat& data, } template -void CFType::GetRecommendations(const size_t numRecs, - arma::Mat& recommendations) +void CFType::GetRecommendations( + const size_t numRecs, + arma::Mat& recommendations) { // Generate list of users. Maybe it would be more efficient to pass an empty // users list, and then have the other overload of GetRecommendations() assume @@ -154,9 +155,10 @@ void CFType::GetRecommendations(const size_t numRecs, } template -void CFType::GetRecommendations(const size_t numRecs, - arma::Mat& recommendations, - const arma::Col& users) +void CFType::GetRecommendations( + const size_t numRecs, + arma::Mat& recommendations, + const arma::Col& users) { // We want to avoid calculating the full rating matrix, so we will do nearest // neighbor search only on the H matrix, using the observation that if the @@ -252,7 +254,7 @@ void CFType::GetRecommendations(const size_t numRecs, // Predict the rating for a single user/item combination. template double CFType::Predict(const size_t user, - const size_t item) const + const size_t item) const { // First, we need to find the nearest neighbors of the given user. // We'll use the same technique as for GetRecommendations(). @@ -297,7 +299,7 @@ double CFType::Predict(const size_t user, // Predict the rating for a group of user/item combinations. template void CFType::Predict(const arma::Mat& combinations, - arma::vec& predictions) const + arma::vec& predictions) const { // First, for nearest neighbor search, stretch the H matrix. arma::mat l = arma::chol(w.t() * w); @@ -354,7 +356,7 @@ void CFType::Predict(const arma::Mat& combinations, template void CFType::CleanData(const arma::mat& data, - arma::sp_mat& cleanedData) + arma::sp_mat& cleanedData) { // Generate list of locations for batch insert constructor for sparse // matrices. @@ -382,7 +384,8 @@ void CFType::CleanData(const arma::mat& data, //! Serialize the model. template template -void CFType::serialize(Archive& ar, const unsigned int /* version */) +void CFType::serialize(Archive& ar, + const unsigned int /* version */) { // This model is simple; just serialize all the members. No special handling // required. diff --git a/src/mlpack/methods/cf/normalization/combined_normalization.hpp b/src/mlpack/methods/cf/normalization/combined_normalization.hpp index dd6b5e76a4..0d34ea25e1 100644 --- a/src/mlpack/methods/cf/normalization/combined_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/combined_normalization.hpp @@ -60,8 +60,8 @@ class CombinedNormalization * @param item Item ID. * @param rating Computed rating before denormalization. */ - double Denormalize(const int user, - const int item, + double Denormalize(const size_t user, + const size_t item, const double rating) const { return SequenceDenormalize<0>(user, item, rating); @@ -141,8 +141,8 @@ class CombinedNormalization template< int I, /* Which normalization in tuple to use */ typename = std::enable_if_t<(I < std::tuple_size::value)>> - double SequenceDenormalize(const int user, - const int item, + double SequenceDenormalize(const size_t user, + const size_t item, const double rating) const { // The order of denormalization should be the reversed order @@ -158,8 +158,8 @@ class CombinedNormalization int I, /* Which normalization in tuple to use */ typename = std::enable_if_t<(I >= std::tuple_size::value)>, typename = void> - double SequenceDenormalize(const int /* user */, - const int /* item */, + double SequenceDenormalize(const size_t /* user */, + const size_t /* item */, const double rating) const { return rating; diff --git a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp index f99e1a566a..09586f2fe8 100644 --- a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp @@ -81,8 +81,8 @@ class ItemMeanNormalization * @param item Item ID. * @param rating Computed rating before denormalization. */ - double Denormalize(const int /* user */, - const int item, + double Denormalize(const size_t /* user */, + const size_t item, const double rating) const { return rating + itemMean(item); diff --git a/src/mlpack/methods/cf/normalization/no_normalization.hpp b/src/mlpack/methods/cf/normalization/no_normalization.hpp index 0e7c970716..ece00fe1f8 100644 --- a/src/mlpack/methods/cf/normalization/no_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/no_normalization.hpp @@ -49,8 +49,8 @@ class NoNormalization * @param item Item ID. * @param rating Computed rating before denormalization. */ - inline double Denormalize(const int /* user */, - const int /* item */, + inline double Denormalize(const size_t /* user */, + const size_t /* item */, const double rating) const { return rating; diff --git a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp index 324916df92..399d10ac4f 100644 --- a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp @@ -64,8 +64,8 @@ class OverallMeanNormalization * @param item Item ID. * @param rating Computed rating before denormalization. */ - double Denormalize(const int /* user */, - const int /* item */, + double Denormalize(const size_t /* user */, + const size_t /* item */, const double rating) const { return rating + mean; diff --git a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp index ad01f7a139..46c1fcd7b9 100644 --- a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp @@ -81,8 +81,8 @@ class UserMeanNormalization * @param item Item ID. * @param rating Computed rating before denormalization. */ - double Denormalize(const int user, - const int /* item */, + double Denormalize(const size_t user, + const size_t /* item */, const double rating) const { return rating + userMean(user); diff --git a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp index 2f5b6b9ab4..15a44906a7 100644 --- a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp @@ -79,8 +79,8 @@ class ZScoreNormalization * @param item Item ID. * @param rating Computed rating before denormalization. */ - double Denormalize(const int /* user */, - const int /* item */, + double Denormalize(const size_t /* user */, + const size_t /* item */, const double rating) const { return rating * stddev + mean; From 4ef5f1810d9f0e40a11318240239aebc0f08031b Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Sun, 3 Jun 2018 21:35:53 +0800 Subject: [PATCH 22/79] minor issues --- src/mlpack/methods/cf/CMakeLists.txt | 1 + src/mlpack/methods/cf/cf_impl.hpp | 2 +- .../normalization/overall_mean_normalization.hpp | 15 ++++++++++----- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/mlpack/methods/cf/CMakeLists.txt b/src/mlpack/methods/cf/CMakeLists.txt index ce03161fd1..98655ab244 100644 --- a/src/mlpack/methods/cf/CMakeLists.txt +++ b/src/mlpack/methods/cf/CMakeLists.txt @@ -8,6 +8,7 @@ set(SOURCES ) add_subdirectory(normalization) +add_subdirectory(decomposition_policies) # Add directory name to sources. set(DIR_SRCS) diff --git a/src/mlpack/methods/cf/cf_impl.hpp b/src/mlpack/methods/cf/cf_impl.hpp index b0b3005803..d9974ff706 100644 --- a/src/mlpack/methods/cf/cf_impl.hpp +++ b/src/mlpack/methods/cf/cf_impl.hpp @@ -63,7 +63,7 @@ CFType::CFType(const MatType& data, this->numUsersForSimilarity = 5; } - Train(data, decomposition, maxIterations, minResidue, mit); + Train(data, decomposition, maxIterations, minResidue, mit); } // Train when data is given in dense matrix form. diff --git a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp index 399d10ac4f..65c3b57882 100644 --- a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp @@ -47,14 +47,19 @@ class OverallMeanNormalization { // Caculate mean of all non zero ratings. if (cleanedData.n_nonzero != 0) + { mean = arma::accu(cleanedData) / cleanedData.n_nonzero; + // Subtract mean from all non zero ratings. + arma::sp_mat::iterator it = cleanedData.begin(); + arma::sp_mat::iterator it_end = cleanedData.end(); + for (; it != it_end; it++) + *it = *it - mean; + } else + { mean = 0; - // Subtract mean from all non zero ratings. - arma::sp_mat::iterator it = cleanedData.begin(); - arma::sp_mat::iterator it_end = cleanedData.end(); - for (; it != it_end; it++) - *it = *it - mean; + // cleanedData remains the same when mean == 0. + } } /** From 062c5c67bd5908b619082b27665e9517db9fd3dd Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Mon, 4 Jun 2018 00:44:19 +0800 Subject: [PATCH 23/79] add some tests --- .../normalization/item_mean_normalization.hpp | 2 +- src/mlpack/tests/cf_test.cpp | 70 +++++++++++++++++-- 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp index 09586f2fe8..b1ee0e0351 100644 --- a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp @@ -37,7 +37,7 @@ class ItemMeanNormalization const size_t itemNum = arma::max(data.row(1)) + 1; itemMean = arma::vec(itemNum, arma::fill::zeros); // Number of ratings for each item. - arma::Row ratingNum(itemNum, arma::fill:zeros); + arma::Row ratingNum(itemNum, arma::fill::zeros); // Sum ratings for each item. data.each_col([&](arma::vec& datapoint) { diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index 31991bd995..74e0b70583 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -18,6 +18,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include @@ -122,7 +128,8 @@ void GetRecommendationsQueriedUser(bool cleanData = true) /** * Make sure recommendations that are generated are reasonably accurate. */ -template +template void RecommendationAccuracy(bool cleanData = true) { DecompositionPolicy decomposition; @@ -168,13 +175,13 @@ void RecommendationAccuracy(bool cleanData = true) { // Make data into sparse matrix. arma::sp_mat cleanedData; - CFType<>::CleanData(dataset, cleanedData); + CFType::CleanData(dataset, cleanedData); arma::sp_mat dataset; dataset = cleanedData; } - CFType<> c(dataset, decomposition, 5, 5, 70); + CFType c(dataset, decomposition, 5, 5, 70); // Obtain 150 recommendations for the users in savedCols, and make sure the // missing item shows up in most of them. First, create the list of users, @@ -223,7 +230,8 @@ void RecommendationAccuracy(bool cleanData = true) } // Make sure that Predict() is returning reasonable results. -template +template void CFPredict(bool cleanData = true) { DecompositionPolicy decomposition; @@ -269,13 +277,13 @@ void CFPredict(bool cleanData = true) { // Make data into sparse matrix. arma::sp_mat cleanedData; - CFType<>::CleanData(dataset, cleanedData); + CFType::CleanData(dataset, cleanedData); arma::sp_mat dataset; dataset = cleanedData; } - CFType<> c(dataset, decomposition, 5, 5, 70); + CFType c(dataset, decomposition, 5, 5, 70); // Now, for each removed rating, make sure the prediction is... reasonably // accurate. @@ -1068,4 +1076,54 @@ BOOST_AUTO_TEST_CASE(SerializationSVDIncompleteTest) Serialization(); } +/** + * Make sure that Predict() is returning reasonable results for NMF and + * OverallMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(CFPredictOverallMeanNormalization) +{ + CFPredict(); +} + +/** + * Make sure that Predict() is returning reasonable results for NMF and + * UserMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(CFPredictUserMeanNormalization) +{ + CFPredict(); +} + +/** + * Make sure that Predict() is returning reasonable results for NMF and + * ItemMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(CFPredictItemMeanNormalization) +{ + CFPredict(); +} + +/** + * Make sure that Predict() is returning reasonable results for NMF and + * ZScoreNormalization. + */ +BOOST_AUTO_TEST_CASE(CFPredictZScoreNormalization) +{ + CFPredict(); +} + +/** + * Make sure that Predict() is returning reasonable results for NMF and + * CombinedNormalization. + */ +BOOST_AUTO_TEST_CASE(CFPredictCombinedNormalization) +{ + CFPredict>(); +} + BOOST_AUTO_TEST_SUITE_END(); From da4bb30d98498a2d7e886f684c1f038fc62fbe04 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Mon, 4 Jun 2018 14:33:00 +0800 Subject: [PATCH 24/79] fix arma version issue --- src/mlpack/methods/cf/normalization/item_mean_normalization.hpp | 2 +- src/mlpack/methods/cf/normalization/user_mean_normalization.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp index b1ee0e0351..ed50ccdf13 100644 --- a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp @@ -66,7 +66,7 @@ class ItemMeanNormalization */ void Normalize(arma::sp_mat& cleanedData) { - itemMean = arma::mean(cleanedData, 1); + itemMean = arma::vec(arma::mean(cleanedData, 1)); arma::sp_mat::iterator it = cleanedData.begin(); arma::sp_mat::iterator it_end = cleanedData.end(); diff --git a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp index 46c1fcd7b9..89472e961a 100644 --- a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp @@ -66,7 +66,7 @@ class UserMeanNormalization */ void Normalize(arma::sp_mat& cleanedData) { - userMean = arma::mean(cleanedData, 0); + userMean = arma::vec(arma::mean(cleanedData, 0)); arma::sp_mat::iterator it = cleanedData.begin(); arma::sp_mat::iterator it_end = cleanedData.end(); From 03e7b1ca9f56ef7da0b8d199a1f2e8bacc75ab05 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Wed, 6 Jun 2018 21:37:27 +0800 Subject: [PATCH 25/79] change totalError to rmse --- src/mlpack/tests/cf_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index 74e0b70583..ac5270f30a 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -296,10 +296,10 @@ void CFPredict(bool cleanData = true) totalError += error; } - totalError = std::sqrt(totalError) / savedCols.n_cols; + const double rmse = std::sqrt(totalError / savedCols.n_cols); - // The mean squared error should be less than one. - BOOST_REQUIRE_LT(totalError, 0.6); + // The root mean square error should be less than ?. + BOOST_REQUIRE_LT(rmse, 4.3); } // Do the same thing as the previous test, but ensure that the ratings we From c20850c8771341f791d3442376200a33ff933155 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Fri, 8 Jun 2018 15:36:50 +0200 Subject: [PATCH 26/79] Introduce run parameter to forward the Forward/Backward/Gradient call inside the merge layer. --- src/mlpack/methods/ann/layer/add_merge.hpp | 46 ++++++++++----- .../methods/ann/layer/add_merge_impl.hpp | 56 ++++++++++++++++-- .../methods/ann/layer/multiply_merge.hpp | 51 ++++++++++++----- .../methods/ann/layer/multiply_merge_impl.hpp | 57 +++++++++++++++++-- .../methods/ann/layer/recurrent_impl.hpp | 4 +- 5 files changed, 178 insertions(+), 36 deletions(-) diff --git a/src/mlpack/methods/ann/layer/add_merge.hpp b/src/mlpack/methods/ann/layer/add_merge.hpp index 44aee7bcac..24120645b0 100644 --- a/src/mlpack/methods/ann/layer/add_merge.hpp +++ b/src/mlpack/methods/ann/layer/add_merge.hpp @@ -46,8 +46,9 @@ class AddMerge * Create the AddMerge object using the specified parameters. * * @param model Expose all the network modules. + * @param run Call the Forward/Backward method before the output is merged. */ - AddMerge(const bool model = false); + AddMerge(const bool model = false, const bool run = true); //! Destructor to release allocated memory. ~AddMerge(); @@ -60,7 +61,7 @@ class AddMerge * @param output Resulting output activation. */ template - void Forward(const InputType&& /* input */, OutputType&& output); + void Forward(InputType&& /* input */, OutputType&& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -77,19 +78,16 @@ class AddMerge arma::Mat&& g); /* - * Add a new module to the model. + * Calculate the gradient using the output delta and the input activation. * - * @param layer The Layer to be added to the model. + * @param input The input parameter used for calculating the gradient. + * @param error The calculated error. + * @param gradient The calculated gradient. */ - void Add(LayerTypes layer) { network.push_back(layer); } - - /* - * Add a new module to the model. - * - * @param layer The Layer to be added to the model. - */ - template - void Add(const LayerType& layer) { network.push_back(new LayerType(layer)); } + template + void Gradient(arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& gradient); /* * Add a new module to the model. @@ -99,6 +97,13 @@ class AddMerge template void Add(Args... args) { network.push_back(new LayerType(args...)); } + /* + * Add a new module to the model. + * + * @param layer The Layer to be added to the model. + */ + void Add(LayerTypes layer) { network.push_back(layer); } + //! Get the input parameter. InputDataType const& InputParameter() const { return inputParameter; } //! Modify the input parameter. @@ -125,6 +130,11 @@ class AddMerge return empty; } + //! Get the parameters. + OutputDataType const& Parameters() const { return weights; } + //! Modify the parameters. + OutputDataType& Parameters() { return weights; } + /** * Serialize the layer. */ @@ -135,6 +145,10 @@ class AddMerge //! Parameter which indicates if the modules should be exposed. bool model; + //! Parameter which indicates if the Forward/Backward method should be called + //! before merging the output. + bool run; + //! We need this to know whether we should delete the layer in the destructor. bool ownsLayer; @@ -156,11 +170,17 @@ class AddMerge //! Locally-stored delta object. OutputDataType delta; + //! Locally-stored gradient object. + OutputDataType gradient; + //! Locally-stored input parameter object. InputDataType inputParameter; //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Locally-stored weight object. + OutputDataType weights; }; // class AddMerge } // namespace ann diff --git a/src/mlpack/methods/ann/layer/add_merge_impl.hpp b/src/mlpack/methods/ann/layer/add_merge_impl.hpp index 1e67c32371..71437d6859 100644 --- a/src/mlpack/methods/ann/layer/add_merge_impl.hpp +++ b/src/mlpack/methods/ann/layer/add_merge_impl.hpp @@ -16,13 +16,18 @@ // In case it hasn't yet been included. #include "add_merge.hpp" +#include "../visitor/forward_visitor.hpp" +#include "../visitor/backward_visitor.hpp" +#include "../visitor/gradient_visitor.hpp" + namespace mlpack { namespace ann /** Artificial Neural Network. */ { template AddMerge::AddMerge( - const bool model) : model(model), ownsLayer(!model) + const bool model, const bool run) : + model(model), run(run), ownsLayer(!model) { // Nothing to do here. } @@ -42,10 +47,19 @@ template template void AddMerge::Forward( - const InputType&& /* input */, OutputType&& output) + InputType&& input, OutputType&& output) { - output = boost::apply_visitor(outputParameterVisitor, network.front()); + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(ForwardVisitor(std::move(input), std::move( + boost::apply_visitor(outputParameterVisitor, network[i]))), + network[i]); + } + } + output = boost::apply_visitor(outputParameterVisitor, network.front()); for (size_t i = 1; i < network.size(); ++i) { output += boost::apply_visitor(outputParameterVisitor, network[i]); @@ -58,7 +72,41 @@ template void AddMerge::Backward( const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { - g = gy; + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( + outputParameterVisitor, network[i])), std::move(gy), std::move( + boost::apply_visitor(deltaVisitor, network[i]))), network[i]); + } + + g = boost::apply_visitor(deltaVisitor, network[0]); + for (size_t i = 1; i < network.size(); ++i) + { + g += boost::apply_visitor(deltaVisitor, network[i]); + } + } + else + g = gy; +} + +template +template +void AddMerge::Gradient( + arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& /* gradient */ ) +{ + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(GradientVisitor(std::move(input), std::move(error)), + network[i]); + } + } } template - void Forward(const InputType&& /* input */, OutputType&& output); + void Forward(InputType&& /* input */, OutputType&& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -77,19 +78,16 @@ class MultiplyMerge arma::Mat&& g); /* - * Add a new module to the model. + * Calculate the gradient using the output delta and the input activation. * - * @param layer The Layer to be added to the model. + * @param input The input parameter used for calculating the gradient. + * @param error The calculated error. + * @param gradient The calculated gradient. */ - void Add(LayerTypes layer) { network.push_back(layer); } - - /* - * Add a new module to the model. - * - * @param layer The Layer to be added to the model. - */ - template - void Add(const LayerType& layer) { network.push_back(new LayerType(layer)); } + template + void Gradient(arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& gradient); /* * Add a new module to the model. @@ -99,6 +97,13 @@ class MultiplyMerge template void Add(Args... args) { network.push_back(new LayerType(args...)); } + /* + * Add a new module to the model. + * + * @param layer The Layer to be added to the model. + */ + void Add(LayerTypes layer) { network.push_back(layer); } + //! Get the input parameter. InputDataType const& InputParameter() const { return inputParameter; } //! Modify the input parameter. @@ -114,6 +119,11 @@ class MultiplyMerge //! Modify the delta. OutputDataType& Delta() { return delta; } + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } + //! Return the model modules. std::vector >& Model() { @@ -125,6 +135,11 @@ class MultiplyMerge return empty; } + //! Get the parameters. + OutputDataType const& Parameters() const { return weights; } + //! Modify the parameters. + OutputDataType& Parameters() { return weights; } + /** * Serialize the layer. */ @@ -135,6 +150,10 @@ class MultiplyMerge //! Parameter which indicates if the modules should be exposed. bool model; + //! Parameter which indicates if the Forward/Backward method should be called + //! before merging the output. + bool run; + //! We need this to know whether we should delete the layer in the destructor. bool ownsLayer; @@ -156,11 +175,17 @@ class MultiplyMerge //! Locally-stored delta object. OutputDataType delta; + //! Locally-stored gradient object. + OutputDataType gradient; + //! Locally-stored input parameter object. InputDataType inputParameter; //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Locally-stored weight object. + OutputDataType weights; }; // class MultiplyMerge } // namespace ann diff --git a/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp b/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp index 6738bac8c4..70660f60ad 100644 --- a/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp +++ b/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp @@ -16,13 +16,18 @@ // In case it hasn't yet been included. #include "multiply_merge.hpp" +#include "../visitor/forward_visitor.hpp" +#include "../visitor/backward_visitor.hpp" +#include "../visitor/gradient_visitor.hpp" + namespace mlpack { namespace ann /** Artificial Neural Network. */ { template MultiplyMerge::MultiplyMerge( - const bool model) : model(model), ownsLayer(!model) + const bool model, const bool run) : + model(model), run(run), ownsLayer(!model) { // Nothing to do here. } @@ -42,10 +47,19 @@ template template void MultiplyMerge::Forward( - const InputType&& /* input */, OutputType&& output) + InputType&& input, OutputType&& output) { - output = boost::apply_visitor(outputParameterVisitor, network.front()); + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(ForwardVisitor(std::move(input), std::move( + boost::apply_visitor(outputParameterVisitor, network[i]))), + network[i]); + } + } + output = boost::apply_visitor(outputParameterVisitor, network.front()); for (size_t i = 1; i < network.size(); ++i) { output %= boost::apply_visitor(outputParameterVisitor, network[i]); @@ -58,7 +72,42 @@ template void MultiplyMerge::Backward( const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { - g = gy; + + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( + outputParameterVisitor, network[i])), std::move(gy), std::move( + boost::apply_visitor(deltaVisitor, network[i]))), network[i]); + } + + g = boost::apply_visitor(deltaVisitor, network[0]); + for (size_t i = 1; i < network.size(); ++i) + { + g += boost::apply_visitor(deltaVisitor, network[i]); + } + } + else + g = gy; +} + +template +template +void MultiplyMerge::Gradient( + arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& /* gradient */ ) +{ + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(GradientVisitor(std::move(input), std::move(error)), + network[i]); + } + } } template::Recurrent( ownsLayer(true) { initialModule = new Sequential<>(); - mergeModule = new AddMerge<>(false); + mergeModule = new AddMerge<>(false, false); recurrentModule = new Sequential<>(false); boost::apply_visitor(AddVisitor(inputModule), @@ -261,7 +261,7 @@ void Recurrent::serialize( if (Archive::is_loading::value) { initialModule = new Sequential<>(); - mergeModule = new AddMerge<>(false); + mergeModule = new AddMerge<>(false, false); recurrentModule = new Sequential<>(false); boost::apply_visitor(AddVisitor(inputModule), From 3e6b64eb426cdc1fdff47c44024b8d9c3151347a Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Fri, 8 Jun 2018 15:39:45 +0200 Subject: [PATCH 27/79] Add merge layer run test. --- src/mlpack/tests/ann_layer_test.cpp | 70 +++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 3 deletions(-) diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index ed52dbd16b..aede1b38c6 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -877,7 +877,7 @@ BOOST_AUTO_TEST_CASE(SimpleAddMergeLayerTest) identityLayer.Forward(std::move(input), std::move(identityLayer.OutputParameter())); - module.Add(identityLayer); + module.Add >(identityLayer); } // Test the Forward function. @@ -1625,7 +1625,7 @@ BOOST_AUTO_TEST_CASE(SimpleMultiplyMergeLayerTest) identityLayer.Forward(std::move(input), std::move(identityLayer.OutputParameter())); - module.Add(identityLayer); + module.Add >(identityLayer); } // Test the Forward function. @@ -1798,4 +1798,68 @@ BOOST_AUTO_TEST_CASE(GradientLayerNormTest) BOOST_REQUIRE_LE(CheckGradient(function), 1e-4); } -BOOST_AUTO_TEST_SUITE_END(); +/** + * Test if the AddMerge layer is able to forward the + * Forward/Backward/Gradient calls. + */ +BOOST_AUTO_TEST_CASE(AddMergeRunTest) +{ + arma::mat output, input, delta, error; + + AddMerge<> module(true, true); + + Linear<>* linear = new Linear<>(10, 10); + module.Add(linear); + + linear->Parameters().randu(); + linear->Reset(); + + input = arma::zeros(10, 1); + module.Forward(std::move(input), std::move(output)); + + double parameterSum = arma::accu(linear->Parameters().submat( + 100, 0, linear->Parameters().n_elem - 1, 0)); + + // Test the Backward function. + module.Backward(std::move(input), std::move(input), std::move(delta)); + + // Clean up before we break, + delete linear; + + BOOST_REQUIRE_CLOSE(parameterSum, arma::accu(output), 1e-3); + BOOST_REQUIRE_EQUAL(arma::accu(delta), 0); +} + +/** + * Test if the MultiplyMerge layer is able to forward the + * Forward/Backward/Gradient calls. + */ +BOOST_AUTO_TEST_CASE(MultiplyMergeRunTest) +{ + arma::mat output, input, delta, error; + + MultiplyMerge<> module(true, true); + + Linear<>* linear = new Linear<>(10, 10); + module.Add(linear); + + linear->Parameters().randu(); + linear->Reset(); + + input = arma::zeros(10, 1); + module.Forward(std::move(input), std::move(output)); + + double parameterSum = arma::accu(linear->Parameters().submat( + 100, 0, linear->Parameters().n_elem - 1, 0)); + + // Test the Backward function. + module.Backward(std::move(input), std::move(input), std::move(delta)); + + // Clean up before we break, + delete linear; + + BOOST_REQUIRE_CLOSE(parameterSum, arma::accu(output), 1e-3); + BOOST_REQUIRE_EQUAL(arma::accu(delta), 0); +} + +BOOST_AUTO_TEST_SUITE_END(); \ No newline at end of file From 3ad61014fe9185d31b81d1afbf8361c4b73f718b Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Fri, 8 Jun 2018 22:27:41 +0800 Subject: [PATCH 28/79] bugfix --- src/mlpack/methods/cf/cf_impl.hpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/cf/cf_impl.hpp b/src/mlpack/methods/cf/cf_impl.hpp index d9974ff706..8aabd45433 100644 --- a/src/mlpack/methods/cf/cf_impl.hpp +++ b/src/mlpack/methods/cf/cf_impl.hpp @@ -76,9 +76,9 @@ void CFType::Train(const arma::mat& data, const bool mit) { // Make a copy of data before performing normalization. - arma::mat ds(data); - normalization.Normalize(ds); - CleanData(ds, cleanedData); + arma::mat normalizedData(data); + normalization.Normalize(normalizedData); + CleanData(normalizedData, cleanedData); // Check if the user wanted us to choose a rank for them. if (rank == 0) @@ -98,7 +98,7 @@ void CFType::Train(const arma::mat& data, // Decompose the data matrix (which is in coordinate list form) to user and // data matrices. Timer::Start("cf_factorization"); - decomposition.Apply(data, cleanedData, rank, w, + decomposition.Apply(normalizedData, cleanedData, rank, w, h, maxIterations, minResidue, mit); Timer::Stop("cf_factorization"); } @@ -112,6 +112,8 @@ void CFType::Train(const arma::sp_mat& data, const double minResidue, const bool mit) { + // data is not used in the following decomposition.Apply() method, so we only + // need to Normalize cleanedData. cleanedData = data; normalization.Normalize(cleanedData); From f96be8b37e258573bc9be955aa4edc5c2b82821c Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Fri, 8 Jun 2018 23:19:48 +0800 Subject: [PATCH 29/79] set zero rating to small double --- .../cf/normalization/item_mean_normalization.hpp | 10 ++++++++++ .../cf/normalization/overall_mean_normalization.hpp | 9 +++++++++ .../cf/normalization/user_mean_normalization.hpp | 10 ++++++++++ .../methods/cf/normalization/z_score_normalization.hpp | 9 +++++++++ 4 files changed, 38 insertions(+) diff --git a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp index ed50ccdf13..6f8aeadfc5 100644 --- a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp @@ -56,6 +56,10 @@ class ItemMeanNormalization data.each_col([&](arma::vec& datapoint) { const size_t item = (size_t) datapoint(1); datapoint(2) -= itemMean(item); + // The algorithm omits rating of zero. If normalized rating equals zero, + // it is set to the smallest positive double value. + if (datapoint(2) == 0) + datapoint(2) = std::numeric_limits::min(); }); } @@ -71,7 +75,13 @@ class ItemMeanNormalization arma::sp_mat::iterator it = cleanedData.begin(); arma::sp_mat::iterator it_end = cleanedData.end(); for (; it != it_end; it++) + { *it = *it - itemMean(it.row()); + // The algorithm omits rating of zero. If normalized rating equals zero, + // it is set to the smallest positive double value. + if (*it == 0) + *it = std::numeric_limits::min(); + } } /** diff --git a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp index 65c3b57882..2b09af435c 100644 --- a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp @@ -36,6 +36,9 @@ class OverallMeanNormalization { mean = arma::mean(data.row(2)); data.row(2) -= mean; + // The algorithm omits rating of zero. If normalized rating equals zero, + // it is set to the smallest positive double value. + data.row(2).replace(0, std::numeric_limits::min()); } /** @@ -53,7 +56,13 @@ class OverallMeanNormalization arma::sp_mat::iterator it = cleanedData.begin(); arma::sp_mat::iterator it_end = cleanedData.end(); for (; it != it_end; it++) + { *it = *it - mean; + // The algorithm omits rating of zero. If normalized rating equals zero, + // it is set to the smallest positive double value. + if (*it == 0) + *it = std::numeric_limits::min(); + } } else { diff --git a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp index 89472e961a..aa9b7b2ff3 100644 --- a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp @@ -56,6 +56,10 @@ class UserMeanNormalization data.each_col([&](arma::vec& datapoint) { const size_t user = (size_t) datapoint(0); datapoint(2) -= userMean(user); + // The algorithm omits rating of zero. If normalized rating equals zero, + // it is set to the smallest positive double value. + if (datapoint(2) == 0) + datapoint(2) = std::numeric_limits::min(); }); } @@ -71,7 +75,13 @@ class UserMeanNormalization arma::sp_mat::iterator it = cleanedData.begin(); arma::sp_mat::iterator it_end = cleanedData.end(); for (; it != it_end; it++) + { *it = *it - userMean(it.col()); + // The algorithm omits rating of zero. If normalized rating equals zero, + // it is set to the smallest positive double value. + if (*it == 0) + *it = std::numeric_limits::min(); + } } /** diff --git a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp index 15a44906a7..977505787c 100644 --- a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp @@ -44,6 +44,9 @@ class ZScoreNormalization } data.row(2) = (data.row(2) - mean) / stddev; + // The algorithm omits rating of zero. If normalized rating equals zero, + // it is set to the smallest positive double value. + data.row(2).replace(0, std::numeric_limits::min()); } /** @@ -69,7 +72,13 @@ class ZScoreNormalization arma::sp_mat::iterator it = cleanedData.begin(); arma::sp_mat::iterator it_end = cleanedData.end(); for (; it != it_end; it++) + { *it = (*it - mean) / stddev; + // The algorithm omits rating of zero. If normalized rating equals zero, + // it is set to the smallest positive double value. + if (*it == 0) + *it = std::numeric_limits::min(); + } } /** From ad90acb8fc842e95b99adbcc080de1d27c019d17 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Sat, 9 Jun 2018 01:02:58 +0200 Subject: [PATCH 30/79] Fix minor style issues (remove blank line, add line at the end of the file). --- src/mlpack/methods/ann/layer/multiply_merge_impl.hpp | 1 - src/mlpack/tests/ann_layer_test.cpp | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp b/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp index 70660f60ad..19d670d113 100644 --- a/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp +++ b/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp @@ -72,7 +72,6 @@ template void MultiplyMerge::Backward( const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { - if (run) { for (size_t i = 0; i < network.size(); ++i) diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index aede1b38c6..0d86ad48cd 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -869,7 +869,7 @@ BOOST_AUTO_TEST_CASE(SimpleAddMergeLayerTest) for (size_t i = 0; i < 5; ++i) { - AddMerge<> module; + AddMerge<> module(false, false); const size_t numMergeModules = math::RandInt(2, 10); for (size_t m = 0; m < numMergeModules; ++m) { @@ -1617,7 +1617,7 @@ BOOST_AUTO_TEST_CASE(SimpleMultiplyMergeLayerTest) for (size_t i = 0; i < 5; ++i) { - MultiplyMerge<> module; + MultiplyMerge<> module(false, false); const size_t numMergeModules = math::RandInt(2, 10); for (size_t m = 0; m < numMergeModules; ++m) { @@ -1862,4 +1862,4 @@ BOOST_AUTO_TEST_CASE(MultiplyMergeRunTest) BOOST_REQUIRE_EQUAL(arma::accu(delta), 0); } -BOOST_AUTO_TEST_SUITE_END(); \ No newline at end of file +BOOST_AUTO_TEST_SUITE_END(); From a33149fa5cfcbbffdebe848c9131e751add14b04 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Sat, 9 Jun 2018 12:03:27 +0800 Subject: [PATCH 31/79] replace() not supported --- .../methods/cf/normalization/overall_mean_normalization.hpp | 5 ++++- .../methods/cf/normalization/z_score_normalization.hpp | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp index 2b09af435c..dd6e7f4b2c 100644 --- a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp @@ -38,7 +38,10 @@ class OverallMeanNormalization data.row(2) -= mean; // The algorithm omits rating of zero. If normalized rating equals zero, // it is set to the smallest positive double value. - data.row(2).replace(0, std::numeric_limits::min()); + data.row(2).for_each([](double& x) { + if (x == 0) + x = std::numeric_limits::min(); + }); } /** diff --git a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp index 977505787c..5875dc78dc 100644 --- a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp @@ -46,7 +46,10 @@ class ZScoreNormalization data.row(2) = (data.row(2) - mean) / stddev; // The algorithm omits rating of zero. If normalized rating equals zero, // it is set to the smallest positive double value. - data.row(2).replace(0, std::numeric_limits::min()); + data.row(2).for_each([](double& x) { + if (x == 0) + x = std::numeric_limits::min(); + }); } /** From 66faebcc2171fdb429e1426f7977128b4bf65438 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Sat, 9 Jun 2018 19:19:07 +0800 Subject: [PATCH 32/79] modify rmse bound in cf_test --- src/mlpack/tests/cf_test.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index ac5270f30a..e71d34c044 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -232,7 +232,7 @@ void RecommendationAccuracy(bool cleanData = true) // Make sure that Predict() is returning reasonable results. template -void CFPredict(bool cleanData = true) +void CFPredict(bool cleanData = true, const double rmseBound = 2.0) { DecompositionPolicy decomposition; // Load the GroupLens dataset; then, we will remove some values from it. @@ -299,7 +299,7 @@ void CFPredict(bool cleanData = true) const double rmse = std::sqrt(totalError / savedCols.n_cols); // The root mean square error should be less than ?. - BOOST_REQUIRE_LT(rmse, 4.3); + BOOST_REQUIRE_LT(rmse, rmseBound); } // Do the same thing as the previous test, but ensure that the ratings we @@ -679,6 +679,8 @@ void Serialization() } } +/** + /** * Make sure that correct number of recommendations are generated when query * set for randomized SVD. @@ -845,7 +847,8 @@ BOOST_AUTO_TEST_CASE(RecommendationAccuracySVDIncompleteTest) // Make sure that Predict() is returning reasonable results for randomized SVD. BOOST_AUTO_TEST_CASE(CFPredictRandSVDTest) { - CFPredict(); + // RandomizedSVD doesn't give a w + CFPredict(true, 4.5); } // Make sure that Predict() is returning reasonable results for regularized SVD. @@ -863,7 +866,7 @@ BOOST_AUTO_TEST_CASE(CFPredictBatchSVDTest) // Make sure that Predict() is returning reasonable results for NMF. BOOST_AUTO_TEST_CASE(CFPredictNMFTest) { - CFPredict(); + CFPredict(true,3.5); } /** @@ -872,7 +875,7 @@ BOOST_AUTO_TEST_CASE(CFPredictNMFTest) */ BOOST_AUTO_TEST_CASE(CFPredictSVDCompleteTest) { - CFPredict(); + CFPredict(true, 3.5); } /** @@ -881,7 +884,7 @@ BOOST_AUTO_TEST_CASE(CFPredictSVDCompleteTest) */ BOOST_AUTO_TEST_CASE(CFPredictSVDIncompleteTest) { - CFPredict(); + CFPredict(true, 3.5); } // Compare batch Predict() and individual Predict() for randomized SVD. From 9963741ffc7cff87f65b70e31aef3518a4004b0d Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Sat, 9 Jun 2018 20:52:27 +0800 Subject: [PATCH 33/79] serialize tuple --- .../normalization/combined_normalization.hpp | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/cf/normalization/combined_normalization.hpp b/src/mlpack/methods/cf/normalization/combined_normalization.hpp index 0d34ea25e1..8cf6062a82 100644 --- a/src/mlpack/methods/cf/normalization/combined_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/combined_normalization.hpp @@ -93,10 +93,9 @@ class CombinedNormalization * Serialization. */ template - void serialize(Archive& ar, const unsigned int /* version */) + void serialize(Archive& ar, const unsigned int version) { - // Boost does not support tuple serialization??? - ar & BOOST_SERIALIZATION_NVP(normalizations); + SequenceSerialize<0, Archive>(ar, version); } private: @@ -185,6 +184,27 @@ class CombinedNormalization typename = void> void SequenceDenormalize(const arma::Mat& /* combinations */, arma::vec& /* predictions */) const { } + + //! Unpack normalizations tuple to serialize. + template< + int I, /* Which normalization in tuple to serialize */ + typename Archive, + typename = std::enable_if_t<(I < std::tuple_size::value)>> + void SequenceSerialize(Archive& ar, const unsigned int version) + { + std::string tagName = "normalization_"; + tagName += std::to_string(I); + ar & boost::serialization::make_nvp(tagName.c_str(), std::get(normalizations)); + SequenceSerialize(ar, version); + } + + //! End of tuple unpacking. + template< + int I, /* Which normalization in tuple to serialize */ + typename Archive, + typename = std::enable_if_t<(I >= std::tuple_size::value)>, + typename = void> + void SequenceSerialize(Archive& /* ar */, const unsigned int /* version */) { } }; } // namespace cf From 061d86e15a46dab2a05d17d7ed93c31a9d7b2383 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Sat, 9 Jun 2018 20:52:43 +0800 Subject: [PATCH 34/79] add tests for normalization --- src/mlpack/tests/cf_test.cpp | 111 +++++++++++++++++++++++++++++++---- 1 file changed, 101 insertions(+), 10 deletions(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index e71d34c044..799c94989a 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -603,7 +603,8 @@ void EmptyConstructorTrain(bool cleanData = true) /** * Ensure we can load and save the CF model. */ -template +template void Serialization() { DecompositionPolicy decomposition; @@ -612,16 +613,16 @@ void Serialization() data::Load("GroupLensSmall.csv", dataset); arma::sp_mat cleanedData; - CFType<>::CleanData(dataset, cleanedData); + CFType::CleanData(dataset, cleanedData); - CFType<> c(cleanedData, decomposition, 5, 5, 70); + CFType c(cleanedData, decomposition, 5, 5, 70); arma::sp_mat randomData; randomData.sprandu(100, 100, 0.3); - CFType<> cXml(randomData, decomposition, 5, 5, 70); - CFType<> cBinary; - CFType<> cText(cleanedData, decomposition, 5, 5, 70); + CFType cXml(randomData, decomposition, 5, 5, 70); + CFType cBinary; + CFType cText(cleanedData, decomposition, 5, 5, 70); SerializeObjectAll(c, cXml, cText, cBinary); @@ -679,8 +680,6 @@ void Serialization() } } -/** - /** * Make sure that correct number of recommendations are generated when query * set for randomized SVD. @@ -847,7 +846,6 @@ BOOST_AUTO_TEST_CASE(RecommendationAccuracySVDIncompleteTest) // Make sure that Predict() is returning reasonable results for randomized SVD. BOOST_AUTO_TEST_CASE(CFPredictRandSVDTest) { - // RandomizedSVD doesn't give a w CFPredict(true, 4.5); } @@ -866,7 +864,7 @@ BOOST_AUTO_TEST_CASE(CFPredictBatchSVDTest) // Make sure that Predict() is returning reasonable results for NMF. BOOST_AUTO_TEST_CASE(CFPredictNMFTest) { - CFPredict(true,3.5); + CFPredict(true, 3.5); } /** @@ -1129,4 +1127,97 @@ BOOST_AUTO_TEST_CASE(CFPredictCombinedNormalization) ItemMeanNormalization>>(); } +/** + * Make sure recommendations that are generated are reasonably accurate + * for OverallMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(RecommendationAccuracyOverallMeanNormalizationTest) +{ + RecommendationAccuracy(); +} + +/** + * Make sure recommendations that are generated are reasonably accurate + * for UserMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(RecommendationAccuracyUserMeanNormalizationTest) +{ + RecommendationAccuracy(); +} + +/** + * Make sure recommendations that are generated are reasonably accurate + * for ItemMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(RecommendationAccuracyItemMeanNormalizationTest) +{ + RecommendationAccuracy(); +} + +/** + * Make sure recommendations that are generated are reasonably accurate + * for ZScoreNormalization. + */ +BOOST_AUTO_TEST_CASE(RecommendationAccuracyZScoreNormalizationTest) +{ + RecommendationAccuracy(); +} + +/** + * Make sure recommendations that are generated are reasonably accurate + * for CombinedNormalization. + */ +BOOST_AUTO_TEST_CASE(RecommendationAccuracyCombinedNormalizationTest) +{ + RecommendationAccuracy>(); +} + +/** + * Ensure we can load and save the CF model using OverallMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(SerializationOverallMeanNormalizationTest) +{ + Serialization(); +} + +/** + * Ensure we can load and save the CF model using UserMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(SerializationUserMeanNormalizationTest) +{ + Serialization(); +} + +/** + * Ensure we can load and save the CF model using ItemMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(SerializationItemMeanNormalizationTest) +{ + Serialization(); +} + +/** + * Ensure we can load and save the CF model using ZScoreMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(SerializationZScoreNormalizationTest) +{ + Serialization(); +} + +/** + * Ensure we can load and save the CF model using CombinedNormalization. + */ +BOOST_AUTO_TEST_CASE(SerializationCombinedNormalizationTest) +{ + Serialization>(); +} + BOOST_AUTO_TEST_SUITE_END(); From 070ffeb84dac03839e8dcc6ee81ea3e4de54a570 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Sat, 9 Jun 2018 21:00:58 +0800 Subject: [PATCH 35/79] comments and styles --- .../methods/cf/normalization/combined_normalization.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/cf/normalization/combined_normalization.hpp b/src/mlpack/methods/cf/normalization/combined_normalization.hpp index 8cf6062a82..fd77dba53d 100644 --- a/src/mlpack/methods/cf/normalization/combined_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/combined_normalization.hpp @@ -194,7 +194,8 @@ class CombinedNormalization { std::string tagName = "normalization_"; tagName += std::to_string(I); - ar & boost::serialization::make_nvp(tagName.c_str(), std::get(normalizations)); + ar & boost::serialization::make_nvp( + tagName.c_str(), std::get(normalizations)); SequenceSerialize(ar, version); } @@ -204,7 +205,8 @@ class CombinedNormalization typename Archive, typename = std::enable_if_t<(I >= std::tuple_size::value)>, typename = void> - void SequenceSerialize(Archive& /* ar */, const unsigned int /* version */) { } + void SequenceSerialize(Archive& /* ar */, const unsigned int /* version */) + { } }; } // namespace cf From 04f0babf5284ce11bf3d925801dcd56f016b674d Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Sun, 10 Jun 2018 17:08:53 +0800 Subject: [PATCH 36/79] update comments --- .../normalization/combined_normalization.hpp | 18 +++++++++++++++++- .../normalization/item_mean_normalization.hpp | 17 ++++++++++++++++- .../overall_mean_normalization.hpp | 13 +++++++++++++ .../normalization/user_mean_normalization.hpp | 16 +++++++++++++++- .../cf/normalization/z_score_normalization.hpp | 12 ++++++++++++ 5 files changed, 73 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/cf/normalization/combined_normalization.hpp b/src/mlpack/methods/cf/normalization/combined_normalization.hpp index fd77dba53d..b172c765f3 100644 --- a/src/mlpack/methods/cf/normalization/combined_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/combined_normalization.hpp @@ -21,8 +21,24 @@ namespace cf { /** * This normalization class performs a sequence of normalization methods on * raw ratings. + * + * An example of how to use CombinedNormalization in CF is shown below: + * + * @code + * extern arma::mat data; // (user, item, rating) table + * extern arma::Col users; // users seeking recommendations + * arma::Mat recommendations; // Recommendations + * + * CFType> cf(data); + * + * // Generate 10 recommendations for all users. + * cf.GetRecommendations(10, recommendations); + * @endcode */ -template +template class CombinedNormalization { public: diff --git a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp index 6f8aeadfc5..49a8aa22c2 100644 --- a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp @@ -20,6 +20,19 @@ namespace cf { /** * This normalization class performs item mean normalization on raw ratings. + * + * An example of how to use ItemMeanNormalization in CF is shown below: + * + * @code + * extern arma::mat data; // (user, item, rating) table + * extern arma::Col users; // users seeking recommendations + * arma::Mat recommendations; // Recommendations + * + * CFType cf(data); + * + * // Generate 10 recommendations for all users. + * cf.GetRecommendations(10, recommendations); + * @endcode */ class ItemMeanNormalization { @@ -49,10 +62,12 @@ class ItemMeanNormalization // Calculate item mean and subtract item mean from ratings. // Set item mean to 0 if the item has no rating. - // Should we use mean of all item means if an item has no rating? for (size_t i = 0; i < itemNum; i++) + { if (ratingNum(i) != 0) itemMean(i) /= ratingNum(i); + } + data.each_col([&](arma::vec& datapoint) { const size_t item = (size_t) datapoint(1); datapoint(2) -= itemMean(item); diff --git a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp index dd6e7f4b2c..13eef18e63 100644 --- a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp @@ -20,6 +20,19 @@ namespace cf { /** * This normalization class performs overall mean normalization on raw ratings. + * + * An example of how to use OverallMeanNormalization in CF is shown below: + * + * @code + * extern arma::mat data; // (user, item, rating) table + * extern arma::Col users; // users seeking recommendations + * arma::Mat recommendations; // Recommendations + * + * CFType cf(data); + * + * // Generate 10 recommendations for all users. + * cf.GetRecommendations(10, recommendations); + * @endcode */ class OverallMeanNormalization { diff --git a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp index aa9b7b2ff3..ca129ca80e 100644 --- a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp @@ -20,6 +20,18 @@ namespace cf { /** * This normalization class performs user mean normalization on raw ratings. + * + * An example of how to use UserMeanNormalization in CF is shown below: + * + * @code + * extern arma::mat data; // (user, item, rating) table + * extern arma::Col users; // users seeking recommendations + * arma::Mat recommendations; // Recommendations + * + * CFType cf(data); + * + * // Generate 10 recommendations for all users. + * cf.GetRecommendations(10, recommendations); */ class UserMeanNormalization { @@ -49,10 +61,12 @@ class UserMeanNormalization // Calculate user mean and subtract user mean from ratings. // Set user mean to 0 if the user has no rating. - // Should we use mean of all user means if a user has no rating? for (size_t i = 0; i < userNum; i++) + { if (ratingNum(i) != 0) userMean(i) /= ratingNum(i); + } + data.each_col([&](arma::vec& datapoint) { const size_t user = (size_t) datapoint(0); datapoint(2) -= userMean(user); diff --git a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp index 5875dc78dc..60ea677da7 100644 --- a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp @@ -19,6 +19,18 @@ namespace cf { /** * This normalization class performs z-score normalization on raw ratings. + * + * An example of how to use ZScoreNormalization in CF is shown below: + * + * @code + * extern arma::mat data; // (user, item, rating) table + * extern arma::Col users; // users seeking recommendations + * arma::Mat recommendations; // Recommendations + * + * CFType cf(data); + * + * // Generate 10 recommendations for all users. + * cf.GetRecommendations(10, recommendations); */ class ZScoreNormalization { From b965b075c65aa106a3093b801aea96ce6f3c8717 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Mon, 11 Jun 2018 21:18:43 +0800 Subject: [PATCH 37/79] update comments & debug --- src/mlpack/methods/cf/cf_impl.hpp | 13 ++++++++++--- .../cf/normalization/combined_normalization.hpp | 4 ++-- .../cf/normalization/item_mean_normalization.hpp | 2 +- .../cf/normalization/user_mean_normalization.hpp | 1 + .../cf/normalization/z_score_normalization.hpp | 1 + 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/mlpack/methods/cf/cf_impl.hpp b/src/mlpack/methods/cf/cf_impl.hpp index 8aabd45433..21917742a3 100644 --- a/src/mlpack/methods/cf/cf_impl.hpp +++ b/src/mlpack/methods/cf/cf_impl.hpp @@ -223,14 +223,17 @@ void CFType::GetRecommendations( for (size_t j = 0; j < averages.n_rows; ++j) { // Ensure that the user hasn't already rated the item. + // The algorithm omits rating of zero. Thus, when normalizing original + // ratings in Normalize(), if normalized rating equals zero, it is set + // to the smallest positive double value. if (cleanedData(j, users(i)) != 0.0) continue; // The user already rated the item. // Is the estimated value better than the worst candidate? - if (averages[j] > pqueue.top().first) + // Denormalize rating before comparison. + double realRating = normalization.Denormalize(i, j, averages[j]); + if (realRating > pqueue.top().first) { - // Denormalize rating before comparation. - double realRating = normalization.Denormalize(i, j, averages[j]); Candidate c = std::make_pair(realRating, j); pqueue.pop(); pqueue.push(c); @@ -370,6 +373,10 @@ void CFType::CleanData(const arma::mat& data, locations(1, i) = ((arma::uword) data(0, i)); locations(0, i) = ((arma::uword) data(1, i)); values(i) = data(2, i); + + // The algorithm omits rating of zero. Thus, when normalizing original + // ratings in Normalize(), if normalized rating equals zero, it is set + // to the smallest positive double value. if (values(i) == 0) Log::Warn << "User rating of 0 ignored for user " << locations(1, i) << ", item " << locations(0, i) << "." << std::endl; diff --git a/src/mlpack/methods/cf/normalization/combined_normalization.hpp b/src/mlpack/methods/cf/normalization/combined_normalization.hpp index b172c765f3..743b19bc8f 100644 --- a/src/mlpack/methods/cf/normalization/combined_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/combined_normalization.hpp @@ -185,7 +185,7 @@ class CombinedNormalization int I, /* Which normalization in tuple to use */ typename = std::enable_if_t<(I < std::tuple_size::value)>> void SequenceDenormalize(const arma::Mat& combinations, - arma::vec& predictions) const + arma::vec& predictions) const { // The order of denormalization should be the reversed order // of normalization. @@ -199,7 +199,7 @@ class CombinedNormalization typename = std::enable_if_t<(I >= std::tuple_size::value)>, typename = void> void SequenceDenormalize(const arma::Mat& /* combinations */, - arma::vec& /* predictions */) const { } + arma::vec& /* predictions */) const { } //! Unpack normalizations tuple to serialize. template< diff --git a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp index 49a8aa22c2..ac79eb8b81 100644 --- a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp @@ -147,7 +147,7 @@ class ItemMeanNormalization } private: - //! item mean. + //! Item mean. arma::vec itemMean; }; diff --git a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp index ca129ca80e..c9bcfe1c44 100644 --- a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp @@ -32,6 +32,7 @@ namespace cf { * * // Generate 10 recommendations for all users. * cf.GetRecommendations(10, recommendations); + * @endcode */ class UserMeanNormalization { diff --git a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp index 60ea677da7..d79318167f 100644 --- a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp @@ -31,6 +31,7 @@ namespace cf { * * // Generate 10 recommendations for all users. * cf.GetRecommendations(10, recommendations); + * @endcode */ class ZScoreNormalization { From beaa319ac591c1224954457e9ea2d1059e833842 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Mon, 11 Jun 2018 21:19:00 +0800 Subject: [PATCH 38/79] remove if(cleanData) block --- src/mlpack/tests/cf_test.cpp | 91 +++++++----------------------------- 1 file changed, 16 insertions(+), 75 deletions(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index 799c94989a..d71a49e613 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -42,7 +42,7 @@ using namespace std; * set. Default case. */ template -void GetRecommendationsAllUsers(bool cleanData = true) +void GetRecommendationsAllUsers() { DecompositionPolicy decomposition; // Dummy number of recommendations. @@ -57,15 +57,6 @@ void GetRecommendationsAllUsers(bool cleanData = true) arma::mat dataset; data::Load("GroupLensSmall.csv", dataset); - if (cleanData) - { - // Make data into sparse matrix. - arma::sp_mat cleanedData; - CFType<>::CleanData(dataset, cleanedData); - - arma::sp_mat dataset; - dataset = cleanedData; - } CFType<> c(dataset, decomposition, 5, 5, 70); // Generate recommendations when query set is not specified. @@ -82,7 +73,7 @@ void GetRecommendationsAllUsers(bool cleanData = true) * Make sure that the recommendations are generated for queried users only. */ template -void GetRecommendationsQueriedUser(bool cleanData = true) +void GetRecommendationsQueriedUser() { DecompositionPolicy decomposition; // Number of users that we will search for recommendations for. @@ -103,16 +94,6 @@ void GetRecommendationsQueriedUser(bool cleanData = true) arma::mat dataset; data::Load("GroupLensSmall.csv", dataset); - if (cleanData) - { - // Make data into sparse matrix. - arma::sp_mat cleanedData; - CFType<>::CleanData(dataset, cleanedData); - - arma::sp_mat dataset; - dataset = cleanedData; - } - CFType<> c(dataset, decomposition, 5, 5, 70); // Generate recommendations when query set is specified. @@ -130,7 +111,7 @@ void GetRecommendationsQueriedUser(bool cleanData = true) */ template -void RecommendationAccuracy(bool cleanData = true) +void RecommendationAccuracy() { DecompositionPolicy decomposition; // Load the GroupLens dataset; then, we will remove some values from it. @@ -171,16 +152,6 @@ void RecommendationAccuracy(bool cleanData = true) } } - if (cleanData) - { - // Make data into sparse matrix. - arma::sp_mat cleanedData; - CFType::CleanData(dataset, cleanedData); - - arma::sp_mat dataset; - dataset = cleanedData; - } - CFType c(dataset, decomposition, 5, 5, 70); // Obtain 150 recommendations for the users in savedCols, and make sure the @@ -232,7 +203,7 @@ void RecommendationAccuracy(bool cleanData = true) // Make sure that Predict() is returning reasonable results. template -void CFPredict(bool cleanData = true, const double rmseBound = 2.0) +void CFPredict(const double rmseBound = 2.0) { DecompositionPolicy decomposition; // Load the GroupLens dataset; then, we will remove some values from it. @@ -273,16 +244,6 @@ void CFPredict(bool cleanData = true, const double rmseBound = 2.0) } } - if (cleanData) - { - // Make data into sparse matrix. - arma::sp_mat cleanedData; - CFType::CleanData(dataset, cleanedData); - - arma::sp_mat dataset; - dataset = cleanedData; - } - CFType c(dataset, decomposition, 5, 5, 70); // Now, for each removed rating, make sure the prediction is... reasonably @@ -306,7 +267,7 @@ void CFPredict(bool cleanData = true, const double rmseBound = 2.0) // predict with the batch Predict() are the same as the individual Predict() // calls. template -void BatchPredict(bool cleanData = true) +void BatchPredict() { DecompositionPolicy decomposition; // Load the GroupLens dataset; then, we will remove some values from it. @@ -347,16 +308,6 @@ void BatchPredict(bool cleanData = true) } } - if (cleanData) - { - // Make data into sparse matrix. - arma::sp_mat cleanedData; - CFType<>::CleanData(dataset, cleanedData); - - arma::sp_mat dataset; - dataset = cleanedData; - } - CFType<> c(dataset, decomposition, 5, 5, 70); // Get predictions for all user/item pairs we held back. @@ -525,7 +476,7 @@ void Train<>(RegSVDPolicy& decomposition) * Make sure we can train a model after using the empty constructor. */ template -void EmptyConstructorTrain(bool cleanData = true) +void EmptyConstructorTrain() { DecompositionPolicy decomposition; // Use default constructor. @@ -569,16 +520,6 @@ void EmptyConstructorTrain(bool cleanData = true) } } - if (cleanData) - { - // Make data into sparse matrix. - arma::sp_mat cleanedData; - CFType<>::CleanData(dataset, cleanedData); - - arma::sp_mat dataset; - dataset = cleanedData; - } - c.Train(dataset, decomposition, 70); // Get predictions for all user/item pairs we held back. @@ -695,7 +636,7 @@ BOOST_AUTO_TEST_CASE(CFGetRecommendationsAllUsersRandSVDTest) */ BOOST_AUTO_TEST_CASE(CFGetRecommendationsAllUsersRegSVDTest) { - GetRecommendationsAllUsers(false); + GetRecommendationsAllUsers(); } /** @@ -750,7 +691,7 @@ BOOST_AUTO_TEST_CASE(CFGetRecommendationsQueriedUserRandSVDTest) */ BOOST_AUTO_TEST_CASE(CFGetRecommendationsQueriedUserRegSVDTest) { - GetRecommendationsQueriedUser(false); + GetRecommendationsQueriedUser(); } /** @@ -804,7 +745,7 @@ BOOST_AUTO_TEST_CASE(RecommendationAccuracyRandSVDTest) */ BOOST_AUTO_TEST_CASE(RecommendationAccuracyRegSVDTest) { - RecommendationAccuracy(false); + RecommendationAccuracy(); } /** @@ -846,13 +787,13 @@ BOOST_AUTO_TEST_CASE(RecommendationAccuracySVDIncompleteTest) // Make sure that Predict() is returning reasonable results for randomized SVD. BOOST_AUTO_TEST_CASE(CFPredictRandSVDTest) { - CFPredict(true, 4.5); + CFPredict(4.5); } // Make sure that Predict() is returning reasonable results for regularized SVD. BOOST_AUTO_TEST_CASE(CFPredictRegSVDTest) { - CFPredict(false); + CFPredict(); } // Make sure that Predict() is returning reasonable results for batch SVD. @@ -864,7 +805,7 @@ BOOST_AUTO_TEST_CASE(CFPredictBatchSVDTest) // Make sure that Predict() is returning reasonable results for NMF. BOOST_AUTO_TEST_CASE(CFPredictNMFTest) { - CFPredict(true, 3.5); + CFPredict(3.5); } /** @@ -873,7 +814,7 @@ BOOST_AUTO_TEST_CASE(CFPredictNMFTest) */ BOOST_AUTO_TEST_CASE(CFPredictSVDCompleteTest) { - CFPredict(true, 3.5); + CFPredict(3.5); } /** @@ -882,7 +823,7 @@ BOOST_AUTO_TEST_CASE(CFPredictSVDCompleteTest) */ BOOST_AUTO_TEST_CASE(CFPredictSVDIncompleteTest) { - CFPredict(true, 3.5); + CFPredict(3.5); } // Compare batch Predict() and individual Predict() for randomized SVD. @@ -894,7 +835,7 @@ BOOST_AUTO_TEST_CASE(CFBatchPredictRandSVDTest) // Compare batch Predict() and individual Predict() for regularized SVD. BOOST_AUTO_TEST_CASE(CFBatchPredictRegSVDTest) { - BatchPredict(false); + BatchPredict(); } // Compare batch Predict() and individual Predict() for batch SVD. @@ -998,7 +939,7 @@ BOOST_AUTO_TEST_CASE(EmptyConstructorTrainRandSVDTest) */ BOOST_AUTO_TEST_CASE(EmptyConstructorTrainRegSVDTest) { - EmptyConstructorTrain(false); + EmptyConstructorTrain(); } /** From 8502230aeea1f894a6e46021e76f0d6caa991966 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Tue, 12 Jun 2018 21:27:42 +0800 Subject: [PATCH 39/79] use complete sentences for examples --- .../methods/cf/normalization/combined_normalization.hpp | 7 ++++--- .../methods/cf/normalization/item_mean_normalization.hpp | 8 +++++--- .../cf/normalization/overall_mean_normalization.hpp | 8 +++++--- .../methods/cf/normalization/user_mean_normalization.hpp | 8 +++++--- .../methods/cf/normalization/z_score_normalization.hpp | 8 +++++--- 5 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/mlpack/methods/cf/normalization/combined_normalization.hpp b/src/mlpack/methods/cf/normalization/combined_normalization.hpp index 743b19bc8f..107005b028 100644 --- a/src/mlpack/methods/cf/normalization/combined_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/combined_normalization.hpp @@ -25,9 +25,10 @@ namespace cf { * An example of how to use CombinedNormalization in CF is shown below: * * @code - * extern arma::mat data; // (user, item, rating) table - * extern arma::Col users; // users seeking recommendations - * arma::Mat recommendations; // Recommendations + * extern arma::mat data; // data is a (user, item, rating) table. + * // Users for whom recommendations are generated. + * extern arma::Col users; + * arma::Mat recommendations; // Resulting recommendations. * * CFType users; // users seeking recommendations - * arma::Mat recommendations; // Recommendations + * extern arma::mat data; // data is a (user, item, rating) table. + * // Users for whom recommendations are generated. + * extern arma::Col users; + * arma::Mat recommendations; // Resulting recommendations. * + * // Use ItemMeanNormalization as normalization method. * CFType cf(data); * * // Generate 10 recommendations for all users. diff --git a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp index 13eef18e63..4c4a6e8fcc 100644 --- a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp @@ -24,10 +24,12 @@ namespace cf { * An example of how to use OverallMeanNormalization in CF is shown below: * * @code - * extern arma::mat data; // (user, item, rating) table - * extern arma::Col users; // users seeking recommendations - * arma::Mat recommendations; // Recommendations + * extern arma::mat data; // data is a (user, item, rating) table. + * // Users for whom recommendations are generated. + * extern arma::Col users; + * arma::Mat recommendations; // Resulting recommendations. * + * // Use OverallMeanNormalization as normalization method. * CFType cf(data); * * // Generate 10 recommendations for all users. diff --git a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp index c9bcfe1c44..b9798ca15a 100644 --- a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp @@ -24,10 +24,12 @@ namespace cf { * An example of how to use UserMeanNormalization in CF is shown below: * * @code - * extern arma::mat data; // (user, item, rating) table - * extern arma::Col users; // users seeking recommendations - * arma::Mat recommendations; // Recommendations + * extern arma::mat data; // data is a (user, item, rating) table. + * // Users for whom recommendations are generated. + * extern arma::Col users; + * arma::Mat recommendations; // Resulting recommendations. * + * // Use UserMeanNormalization as normalization method. * CFType cf(data); * * // Generate 10 recommendations for all users. diff --git a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp index d79318167f..1f68e5f532 100644 --- a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp @@ -23,10 +23,12 @@ namespace cf { * An example of how to use ZScoreNormalization in CF is shown below: * * @code - * extern arma::mat data; // (user, item, rating) table - * extern arma::Col users; // users seeking recommendations - * arma::Mat recommendations; // Recommendations + * extern arma::mat data; // data is a (user, item, rating) table. + * // Users for whom recommendations are generated. + * extern arma::Col users; + * arma::Mat recommendations; // Resulting recommendations. * + * // Use ZScoreNormalization as normalization method. * CFType cf(data); * * // Generate 10 recommendations for all users. From de673717c12a59d7c42af2d24435817a9c18b316 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Tue, 12 Jun 2018 21:36:07 +0800 Subject: [PATCH 40/79] change method names to Mean() and return const refenrence --- .../methods/cf/normalization/item_mean_normalization.hpp | 5 +---- .../methods/cf/normalization/user_mean_normalization.hpp | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp index 35d7292157..9f44fb06ed 100644 --- a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp @@ -134,10 +134,7 @@ class ItemMeanNormalization /** * Return item mean. */ - arma::vec ItemMean() const - { - return itemMean; - } + const arma::vec& Mean() const { return itemMean; } /** * Serialization. diff --git a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp index b9798ca15a..8b524c87db 100644 --- a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp @@ -134,10 +134,7 @@ class UserMeanNormalization /** * Return user mean. */ - arma::vec UserMean() const - { - return userMean; - } + const arma::vec& Mean() const { return userMean; } /** * Serialization. From f2d73d5390397ba04cef2815eaeb5ded04cad535 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Tue, 12 Jun 2018 21:58:19 +0800 Subject: [PATCH 41/79] change param specification --- src/mlpack/methods/cf/normalization/combined_normalization.hpp | 2 +- src/mlpack/methods/cf/normalization/item_mean_normalization.hpp | 2 +- src/mlpack/methods/cf/normalization/no_normalization.hpp | 2 +- .../methods/cf/normalization/overall_mean_normalization.hpp | 2 +- src/mlpack/methods/cf/normalization/user_mean_normalization.hpp | 2 +- src/mlpack/methods/cf/normalization/z_score_normalization.hpp | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/mlpack/methods/cf/normalization/combined_normalization.hpp b/src/mlpack/methods/cf/normalization/combined_normalization.hpp index 107005b028..d65a0c4cf8 100644 --- a/src/mlpack/methods/cf/normalization/combined_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/combined_normalization.hpp @@ -61,7 +61,7 @@ class CombinedNormalization /** * Normalize the data by calling Normalize() in each normalization object. * - * @param cleanedData Sparse matrix data. + * @param cleanedData Input data as a sparse matrix. */ void Normalize(arma::sp_mat& cleanedData) { diff --git a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp index 9f44fb06ed..3baafd5d81 100644 --- a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp @@ -83,7 +83,7 @@ class ItemMeanNormalization /** * Normalize the data by subtracting item mean from each of existing ratings. * - * @param cleanedData Sparse matrix data. + * @param cleanedData Input data as a sparse matrix. */ void Normalize(arma::sp_mat& cleanedData) { diff --git a/src/mlpack/methods/cf/normalization/no_normalization.hpp b/src/mlpack/methods/cf/normalization/no_normalization.hpp index ece00fe1f8..211718ef48 100644 --- a/src/mlpack/methods/cf/normalization/no_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/no_normalization.hpp @@ -38,7 +38,7 @@ class NoNormalization /** * Do nothing. * - * @param cleanedData Sparse matrix data. + * @param cleanedData Input data as a sparse matrix. */ inline void Normalize(const arma::sp_mat& /* cleanedData */) const { } diff --git a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp index 4c4a6e8fcc..3de59057d6 100644 --- a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp @@ -62,7 +62,7 @@ class OverallMeanNormalization /** * Normalize the data by subtracting the mean of all existing ratings. * - * @param cleanedData Sparse matrix data. + * @param cleanedData Input data as a sparse matrix. */ void Normalize(arma::sp_mat& cleanedData) { diff --git a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp index 8b524c87db..39f382c11d 100644 --- a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp @@ -83,7 +83,7 @@ class UserMeanNormalization /** * Normalize the data by subtracting user mean from each of existing rating. * - * @param cleanedData Sparse matrix data. + * @param cleanedData Input data as a sparse matrix. */ void Normalize(arma::sp_mat& cleanedData) { diff --git a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp index 1f68e5f532..8405ced3c5 100644 --- a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp @@ -70,7 +70,7 @@ class ZScoreNormalization /** * Normalize the data to zero mean and one standard deviation. * - * @param cleanedData Sparse matrix data. + * @param cleanedData Input data as a sparse matrix. */ void Normalize(arma::sp_mat& cleanedData) { From e0544b5e2360feb85fb9b056db33360097916269 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Tue, 12 Jun 2018 22:31:23 +0800 Subject: [PATCH 42/79] new line for brace --- .../methods/cf/normalization/item_mean_normalization.hpp | 6 ++++-- .../methods/cf/normalization/overall_mean_normalization.hpp | 3 ++- .../methods/cf/normalization/user_mean_normalization.hpp | 6 ++++-- .../methods/cf/normalization/z_score_normalization.hpp | 3 ++- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp index 3baafd5d81..c4373975b4 100644 --- a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp @@ -55,7 +55,8 @@ class ItemMeanNormalization arma::Row ratingNum(itemNum, arma::fill::zeros); // Sum ratings for each item. - data.each_col([&](arma::vec& datapoint) { + data.each_col([&](arma::vec& datapoint) + { const size_t item = (size_t) datapoint(1); const double rating = datapoint(2); itemMean(item) += rating; @@ -70,7 +71,8 @@ class ItemMeanNormalization itemMean(i) /= ratingNum(i); } - data.each_col([&](arma::vec& datapoint) { + data.each_col([&](arma::vec& datapoint) + { const size_t item = (size_t) datapoint(1); datapoint(2) -= itemMean(item); // The algorithm omits rating of zero. If normalized rating equals zero, diff --git a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp index 3de59057d6..6891421c6d 100644 --- a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp @@ -53,7 +53,8 @@ class OverallMeanNormalization data.row(2) -= mean; // The algorithm omits rating of zero. If normalized rating equals zero, // it is set to the smallest positive double value. - data.row(2).for_each([](double& x) { + data.row(2).for_each([](double& x) + { if (x == 0) x = std::numeric_limits::min(); }); diff --git a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp index 39f382c11d..382fb4566e 100644 --- a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp @@ -55,7 +55,8 @@ class UserMeanNormalization arma::Row ratingNum(userNum, arma::fill::zeros); // Sum ratings for each user. - data.each_col([&](arma::vec& datapoint) { + data.each_col([&](arma::vec& datapoint) + { const size_t user = (size_t) datapoint(0); const double rating = datapoint(2); userMean(user) += rating; @@ -70,7 +71,8 @@ class UserMeanNormalization userMean(i) /= ratingNum(i); } - data.each_col([&](arma::vec& datapoint) { + data.each_col([&](arma::vec& datapoint) + { const size_t user = (size_t) datapoint(0); datapoint(2) -= userMean(user); // The algorithm omits rating of zero. If normalized rating equals zero, diff --git a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp index 8405ced3c5..00ec50fc14 100644 --- a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp @@ -61,7 +61,8 @@ class ZScoreNormalization data.row(2) = (data.row(2) - mean) / stddev; // The algorithm omits rating of zero. If normalized rating equals zero, // it is set to the smallest positive double value. - data.row(2).for_each([](double& x) { + data.row(2).for_each([](double& x) + { if (x == 0) x = std::numeric_limits::min(); }); From 98a1f64e0fb0c171fcba6631081b4e1293bc64a7 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Tue, 12 Jun 2018 22:42:02 +0800 Subject: [PATCH 43/79] templatize Normalize() functions in some classes --- .../normalization/combined_normalization.hpp | 38 ++++--------------- .../cf/normalization/no_normalization.hpp | 12 ++---- 2 files changed, 10 insertions(+), 40 deletions(-) diff --git a/src/mlpack/methods/cf/normalization/combined_normalization.hpp b/src/mlpack/methods/cf/normalization/combined_normalization.hpp index d65a0c4cf8..bddc76520c 100644 --- a/src/mlpack/methods/cf/normalization/combined_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/combined_normalization.hpp @@ -51,23 +51,14 @@ class CombinedNormalization /** * Normalize the data by calling Normalize() in each normalization object. * - * @param data Input dataset in the form of coordinate list. + * @param data Input dataset. */ - void Normalize(arma::mat& data) + template + void Normalize(MatType& data) { SequenceNormalize<0>(data); } - /** - * Normalize the data by calling Normalize() in each normalization object. - * - * @param cleanedData Input data as a sparse matrix. - */ - void Normalize(arma::sp_mat& cleanedData) - { - SequenceNormalize<0>(cleanedData); - } - /** * Denormalize rating by calling Denormalize() in each normalization object. * Note that the order of objects calling Denormalize() should be the @@ -122,8 +113,9 @@ class CombinedNormalization //! Unpack normalizations tuple to normalize data. template< int I, /* Which normalization in tuple to use */ + typename MatType, typename = std::enable_if_t<(I < std::tuple_size::value)>> - void SequenceNormalize(arma::mat& data) + void SequenceNormalize(MatType& data) { std::get(normalizations).Normalize(data); SequenceNormalize(data); @@ -132,26 +124,10 @@ class CombinedNormalization //! End of tuple unpacking. template< int I, /* Which normalization in tuple to use */ + typename MatType, typename = std::enable_if_t<(I >= std::tuple_size::value)>, typename = void> - void SequenceNormalize(arma::mat& /* data */) { } - - //! Unpack normalizations tuple to normalize cleanedData. - template< - int I, /* Which normalization in tuple to use */ - typename = std::enable_if_t<(I < std::tuple_size::value)>> - void SequenceNormalize(arma::sp_mat& cleanedData) - { - std::get(normalizations).Normalize(cleanedData); - SequenceNormalize(cleanedData); - } - - //! End of tuple unpacking. - template< - int I, /* Which normalization in tuple to use */ - typename = std::enable_if_t<(I >= std::tuple_size::value)>, - typename = void> - void SequenceNormalize(arma::sp_mat& /* cleanedData */) { } + void SequenceNormalize(MatType& /* data */) { } //! Unpack normalizations tuple to denormalize. template< diff --git a/src/mlpack/methods/cf/normalization/no_normalization.hpp b/src/mlpack/methods/cf/normalization/no_normalization.hpp index 211718ef48..cd070d4a60 100644 --- a/src/mlpack/methods/cf/normalization/no_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/no_normalization.hpp @@ -31,16 +31,10 @@ class NoNormalization /** * Do nothing. * - * @param data Input dataset in the form of coordinate list. + * @param data Input dataset. */ - inline void Normalize(const arma::mat& /* data */) const { } - - /** - * Do nothing. - * - * @param cleanedData Input data as a sparse matrix. - */ - inline void Normalize(const arma::sp_mat& /* cleanedData */) const { } + template + inline void Normalize(const MatType& /* data */) const { } /** * Do nothing. From 7d4055b03609d29fd26f191e36bbe8436d7163ec Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Wed, 13 Jun 2018 08:43:36 +0800 Subject: [PATCH 44/79] initialize members in constructor --- .../methods/cf/normalization/overall_mean_normalization.hpp | 2 +- src/mlpack/methods/cf/normalization/z_score_normalization.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp index 6891421c6d..060832668b 100644 --- a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp @@ -40,7 +40,7 @@ class OverallMeanNormalization { public: // Empty constructor. - OverallMeanNormalization() { } + OverallMeanNormalization(): mean(0) { } /** * Normalize the data by subtracting the mean of all existing ratings. diff --git a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp index 00ec50fc14..8da13da102 100644 --- a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp @@ -39,7 +39,7 @@ class ZScoreNormalization { public: // Empty constructor. - ZScoreNormalization() { } + ZScoreNormalization(): mean(0), stddev(1) { } /** * Normalize the data to zero mean and one standard deviation. From 17860e993663eeeb5beb27b8fb87d021784e107e Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Wed, 13 Jun 2018 08:45:26 +0800 Subject: [PATCH 45/79] style --- .../methods/cf/normalization/overall_mean_normalization.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp index 060832668b..4dab41fbc4 100644 --- a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp @@ -40,7 +40,7 @@ class OverallMeanNormalization { public: // Empty constructor. - OverallMeanNormalization(): mean(0) { } + OverallMeanNormalization() : mean(0) { } /** * Normalize the data by subtracting the mean of all existing ratings. From b7a874cee2f3838df40a01b7c02dfc4b3a5b1623 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Thu, 14 Jun 2018 21:13:19 +0800 Subject: [PATCH 46/79] style fix --- src/mlpack/methods/cf/cf.hpp | 18 +++++++++++++----- src/mlpack/methods/cf/cf_impl.hpp | 4 ++-- .../cf/normalization/z_score_normalization.hpp | 2 +- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/mlpack/methods/cf/cf.hpp b/src/mlpack/methods/cf/cf.hpp index 4134eb37cb..a3f9f0d4ba 100644 --- a/src/mlpack/methods/cf/cf.hpp +++ b/src/mlpack/methods/cf/cf.hpp @@ -60,9 +60,6 @@ namespace cf /** Collaborative filtering. **/ { * @tparam NormalizationType The type of normalization performed on raw data. * Data is normalized before calling Train() method. Predicted rating is * denormalized before return. - * - * @tparam DecompositionPolicy The algorithm to use to decompose - * the rating matrix (a W and H matrix). */ template class CFType @@ -72,8 +69,7 @@ class CFType * Initialize the CFType object without performing any factorization. Be sure to * call Train() before calling GetRecommendations() or any other functions! */ - CFType(const size_t numUsersForSimilarity = 5, - const size_t rank = 0); + CFType(const size_t numUsersForSimilarity = 5, const size_t rank = 0); /** * Initialize the CFType object using any decomposition method, immediately @@ -86,6 +82,12 @@ class CFType * where each column corresponds to a (user, item, rating) entry in the * matrix or a sparse matrix representing (user, item) table. * + * @tparam MatType The type of input matrix, which is expected to be either + * arma::mat (table of (user, item, rating)) or arma::sp_mat (sparse + * rating matrix where row is item and column is user). + * @tparam DecompositionPolicy The algorithm to use to decompose + * the rating matrix (a W and H matrix). + * * @param data Data matrix: dense matrix (coordinate lists) * or sparse matrix(cleaned). * @param decomposition Instantiated DecompositionPolicy object. @@ -109,6 +111,9 @@ class CFType * parameters that have already been set for the model (specifically, the rank * parameter), and optionally, using the given DecompositionPolicy. * + * @tparam DecompositionPolicy The algorithm to use to decompose + * the rating matrix (a W and H matrix). + * * @param data Input dataset; dense matrix (coordinate lists). * @param decomposition Instantiated DecompositionPolicy object. * @param maxIterations Maximum number of iterations. @@ -127,6 +132,9 @@ class CFType * parameters that have already been set for the model (specifically, the * rank parameter), and optionally, using the given DecompositionPolicy. * + * @tparam DecompositionPolicy The algorithm to use to decompose + * the rating matrix (a W and H matrix). + * * @param data Input dataset; sparse matrix (user item table). * @param decomposition Instantiated DecompositionPolicy object. * @param maxIterations Maximum number of iterations. diff --git a/src/mlpack/methods/cf/cf_impl.hpp b/src/mlpack/methods/cf/cf_impl.hpp index 21917742a3..fb5a7ae31a 100644 --- a/src/mlpack/methods/cf/cf_impl.hpp +++ b/src/mlpack/methods/cf/cf_impl.hpp @@ -26,8 +26,8 @@ namespace cf { template CFType::CFType(const size_t numUsersForSimilarity, const size_t rank) : - numUsersForSimilarity(numUsersForSimilarity), - rank(rank) + numUsersForSimilarity(numUsersForSimilarity), + rank(rank) { // Validate neighbourhood size. if (numUsersForSimilarity < 1) diff --git a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp index 8da13da102..4ae8b22920 100644 --- a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp @@ -39,7 +39,7 @@ class ZScoreNormalization { public: // Empty constructor. - ZScoreNormalization(): mean(0), stddev(1) { } + ZScoreNormalization() : mean(0), stddev(1) { } /** * Normalize the data to zero mean and one standard deviation. From 5fbe03bc6ef0bb0d0416267d4f2e284c0f186718 Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Thu, 14 Jun 2018 21:21:14 +0800 Subject: [PATCH 47/79] Denormalize(users(i), ...) --- src/mlpack/methods/cf/cf_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/cf/cf_impl.hpp b/src/mlpack/methods/cf/cf_impl.hpp index fb5a7ae31a..64a128d417 100644 --- a/src/mlpack/methods/cf/cf_impl.hpp +++ b/src/mlpack/methods/cf/cf_impl.hpp @@ -231,7 +231,7 @@ void CFType::GetRecommendations( // Is the estimated value better than the worst candidate? // Denormalize rating before comparison. - double realRating = normalization.Denormalize(i, j, averages[j]); + double realRating = normalization.Denormalize(users(i), j, averages[j]); if (realRating > pqueue.top().first) { Candidate c = std::make_pair(realRating, j); From 2554f600f7ec6a9f45d35589950dd61b9779cd8a Mon Sep 17 00:00:00 2001 From: Wenhao-H Date: Thu, 14 Jun 2018 21:28:23 +0800 Subject: [PATCH 48/79] change from arma::vec userMean to arma::rowvec userMean --- .../methods/cf/normalization/user_mean_normalization.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp index 382fb4566e..de1fde589d 100644 --- a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp +++ b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp @@ -50,7 +50,7 @@ class UserMeanNormalization void Normalize(arma::mat& data) { const size_t userNum = arma::max(data.row(0)) + 1; - userMean = arma::vec(userNum, arma::fill::zeros); + userMean = arma::rowvec(userNum, arma::fill::zeros); // Number of ratings for each user. arma::Row ratingNum(userNum, arma::fill::zeros); @@ -89,7 +89,7 @@ class UserMeanNormalization */ void Normalize(arma::sp_mat& cleanedData) { - userMean = arma::vec(arma::mean(cleanedData, 0)); + userMean = arma::rowvec(arma::mean(cleanedData, 0)); arma::sp_mat::iterator it = cleanedData.begin(); arma::sp_mat::iterator it_end = cleanedData.end(); @@ -136,7 +136,7 @@ class UserMeanNormalization /** * Return user mean. */ - const arma::vec& Mean() const { return userMean; } + const arma::rowvec& Mean() const { return userMean; } /** * Serialization. @@ -149,7 +149,7 @@ class UserMeanNormalization private: //! User mean. - arma::vec userMean; + arma::rowvec userMean; }; } // namespace cf From a011e7e48eb6207f0962605cdbe3ec148c4c72bd Mon Sep 17 00:00:00 2001 From: akhandait Date: Sat, 16 Jun 2018 11:57:42 +0530 Subject: [PATCH 49/79] removed reduntant data members, moved NegativeLogLikelihood to loss folder --- src/mlpack/methods/ann/layer/CMakeLists.txt | 2 - src/mlpack/methods/ann/layer/add.hpp | 8 -- .../methods/ann/layer/alpha_dropout.hpp | 8 -- .../methods/ann/layer/atrous_convolution.hpp | 8 -- src/mlpack/methods/ann/layer/base_layer.hpp | 8 -- src/mlpack/methods/ann/layer/batch_norm.hpp | 8 -- .../ann/layer/bilinear_interpolation.hpp | 7 -- .../methods/ann/layer/concat_performance.hpp | 8 -- src/mlpack/methods/ann/layer/constant.hpp | 8 -- src/mlpack/methods/ann/layer/dropconnect.hpp | 8 -- src/mlpack/methods/ann/layer/dropout.hpp | 8 -- src/mlpack/methods/ann/layer/elu.hpp | 8 -- src/mlpack/methods/ann/layer/fast_lstm.hpp | 8 -- .../methods/ann/layer/flexible_relu.hpp | 8 -- src/mlpack/methods/ann/layer/glimpse.hpp | 8 -- src/mlpack/methods/ann/layer/gru.hpp | 8 -- src/mlpack/methods/ann/layer/hard_tanh.hpp | 8 -- src/mlpack/methods/ann/layer/join.hpp | 8 -- src/mlpack/methods/ann/layer/layer_norm.hpp | 8 -- src/mlpack/methods/ann/layer/leaky_relu.hpp | 8 -- src/mlpack/methods/ann/layer/log_softmax.hpp | 8 -- src/mlpack/methods/ann/layer/lookup.hpp | 8 -- src/mlpack/methods/ann/layer/lstm.hpp | 8 -- src/mlpack/methods/ann/layer/max_pooling.hpp | 8 -- src/mlpack/methods/ann/layer/mean_pooling.hpp | 8 -- .../methods/ann/layer/multiply_constant.hpp | 8 -- .../methods/ann/layer/multiply_merge.hpp | 8 -- .../ann/layer/negative_log_likelihood.hpp | 106 ------------------ .../layer/negative_log_likelihood_impl.hpp | 75 ------------- .../methods/ann/layer/parametric_relu.hpp | 8 -- src/mlpack/methods/ann/layer/recurrent.hpp | 8 -- .../methods/ann/layer/recurrent_attention.hpp | 8 -- .../methods/ann/layer/reinforce_normal.hpp | 8 -- src/mlpack/methods/ann/layer/select.hpp | 8 -- src/mlpack/methods/ann/layer/sequential.hpp | 3 +- src/mlpack/methods/ann/layer/subview.hpp | 8 -- .../methods/ann/layer/vr_class_reward.hpp | 8 -- .../methods/ann/loss_functions/CMakeLists.txt | 2 + .../loss_functions/cross_entropy_error.hpp | 16 --- .../ann/loss_functions/kl_divergence.hpp | 16 --- .../ann/loss_functions/mean_squared_error.hpp | 16 --- .../sigmoid_cross_entropy_error.hpp | 16 --- 42 files changed, 4 insertions(+), 511 deletions(-) delete mode 100644 src/mlpack/methods/ann/layer/negative_log_likelihood.hpp delete mode 100644 src/mlpack/methods/ann/layer/negative_log_likelihood_impl.hpp diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt index 6523499f74..3ca22555f6 100644 --- a/src/mlpack/methods/ann/layer/CMakeLists.txt +++ b/src/mlpack/methods/ann/layer/CMakeLists.txt @@ -65,8 +65,6 @@ set(SOURCES multiply_constant_impl.hpp multiply_merge.hpp multiply_merge_impl.hpp - negative_log_likelihood.hpp - negative_log_likelihood_impl.hpp parametric_relu.hpp parametric_relu_impl.hpp recurrent.hpp diff --git a/src/mlpack/methods/ann/layer/add.hpp b/src/mlpack/methods/ann/layer/add.hpp index 1db7e95a3e..47de360eed 100644 --- a/src/mlpack/methods/ann/layer/add.hpp +++ b/src/mlpack/methods/ann/layer/add.hpp @@ -82,11 +82,6 @@ class Add //! Modify the parameters. OutputDataType& Parameters() { return weights; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -121,9 +116,6 @@ class Add //! Locally-stored gradient object. OutputDataType gradient; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class Add diff --git a/src/mlpack/methods/ann/layer/alpha_dropout.hpp b/src/mlpack/methods/ann/layer/alpha_dropout.hpp index 978edd861d..dc1640c54f 100644 --- a/src/mlpack/methods/ann/layer/alpha_dropout.hpp +++ b/src/mlpack/methods/ann/layer/alpha_dropout.hpp @@ -80,11 +80,6 @@ class AlphaDropout arma::Mat&& gy, arma::Mat&& g); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -134,9 +129,6 @@ class AlphaDropout //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/atrous_convolution.hpp b/src/mlpack/methods/ann/layer/atrous_convolution.hpp index 79bf1def2d..e41cf1613e 100644 --- a/src/mlpack/methods/ann/layer/atrous_convolution.hpp +++ b/src/mlpack/methods/ann/layer/atrous_convolution.hpp @@ -130,11 +130,6 @@ class AtrousConvolution //! Modify the parameters. OutputDataType& Parameters() { return weights; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -344,9 +339,6 @@ class AtrousConvolution //! Locally-stored gradient object. OutputDataType gradient; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class AtrousConvolution diff --git a/src/mlpack/methods/ann/layer/base_layer.hpp b/src/mlpack/methods/ann/layer/base_layer.hpp index bcb230f3f4..780df1f1a4 100644 --- a/src/mlpack/methods/ann/layer/base_layer.hpp +++ b/src/mlpack/methods/ann/layer/base_layer.hpp @@ -88,11 +88,6 @@ class BaseLayer g = gy % derivative; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -116,9 +111,6 @@ class BaseLayer //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class BaseLayer diff --git a/src/mlpack/methods/ann/layer/batch_norm.hpp b/src/mlpack/methods/ann/layer/batch_norm.hpp index 662619ef80..9b3932e196 100644 --- a/src/mlpack/methods/ann/layer/batch_norm.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm.hpp @@ -112,11 +112,6 @@ class BatchNorm //! Modify the parameters. OutputDataType& Parameters() { return weights; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -186,9 +181,6 @@ class BatchNorm //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/bilinear_interpolation.hpp b/src/mlpack/methods/ann/layer/bilinear_interpolation.hpp index 11305648b9..2d250dcf8d 100644 --- a/src/mlpack/methods/ann/layer/bilinear_interpolation.hpp +++ b/src/mlpack/methods/ann/layer/bilinear_interpolation.hpp @@ -82,11 +82,6 @@ class BilinearInterpolation arma::Mat&& gradient, arma::Mat&& output); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -116,8 +111,6 @@ class BilinearInterpolation size_t depth; //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class BilinearInterpolation diff --git a/src/mlpack/methods/ann/layer/concat_performance.hpp b/src/mlpack/methods/ann/layer/concat_performance.hpp index 121bc2a7f1..02ba31a683 100644 --- a/src/mlpack/methods/ann/layer/concat_performance.hpp +++ b/src/mlpack/methods/ann/layer/concat_performance.hpp @@ -72,11 +72,6 @@ class ConcatPerformance const arma::Mat&& target, arma::Mat&& output); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -103,9 +98,6 @@ class ConcatPerformance //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class ConcatPerformance diff --git a/src/mlpack/methods/ann/layer/constant.hpp b/src/mlpack/methods/ann/layer/constant.hpp index 03d5a90144..62ed80faa9 100644 --- a/src/mlpack/methods/ann/layer/constant.hpp +++ b/src/mlpack/methods/ann/layer/constant.hpp @@ -66,11 +66,6 @@ class Constant DataType&& /* gy */, DataType&& g); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -100,9 +95,6 @@ class Constant //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class ConstantLayer diff --git a/src/mlpack/methods/ann/layer/dropconnect.hpp b/src/mlpack/methods/ann/layer/dropconnect.hpp index 46bef506fd..1ed6f2ce0c 100644 --- a/src/mlpack/methods/ann/layer/dropconnect.hpp +++ b/src/mlpack/methods/ann/layer/dropconnect.hpp @@ -120,11 +120,6 @@ class DropConnect //! Modify the parameters. OutputDataType& Parameters() { return parameters; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -178,9 +173,6 @@ class DropConnect //! Locally-stored gradient object. OutputDataType gradient; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/dropout.hpp b/src/mlpack/methods/ann/layer/dropout.hpp index a388820c71..696d92869c 100644 --- a/src/mlpack/methods/ann/layer/dropout.hpp +++ b/src/mlpack/methods/ann/layer/dropout.hpp @@ -80,11 +80,6 @@ class Dropout arma::Mat&& gy, arma::Mat&& g); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -120,9 +115,6 @@ class Dropout //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/elu.hpp b/src/mlpack/methods/ann/layer/elu.hpp index 06d6253313..5e273a6dc1 100644 --- a/src/mlpack/methods/ann/layer/elu.hpp +++ b/src/mlpack/methods/ann/layer/elu.hpp @@ -144,11 +144,6 @@ class ELU template void Backward(const DataType&& input, DataType&& gy, DataType&& g); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -238,9 +233,6 @@ class ELU //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/fast_lstm.hpp b/src/mlpack/methods/ann/layer/fast_lstm.hpp index e853bb16d2..034bafd744 100644 --- a/src/mlpack/methods/ann/layer/fast_lstm.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm.hpp @@ -138,11 +138,6 @@ class FastLSTM //! Modify the parameters. OutputDataType& Parameters() { return weights; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -254,9 +249,6 @@ class FastLSTM //! Locally-stored gradient object. OutputDataType grad; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/flexible_relu.hpp b/src/mlpack/methods/ann/layer/flexible_relu.hpp index a9c837e022..c5e69bc7f8 100644 --- a/src/mlpack/methods/ann/layer/flexible_relu.hpp +++ b/src/mlpack/methods/ann/layer/flexible_relu.hpp @@ -115,11 +115,6 @@ class FlexibleReLU //! Modify the parameters. OutputDataType& Parameters() { return alpha; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -150,9 +145,6 @@ class FlexibleReLU //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/glimpse.hpp b/src/mlpack/methods/ann/layer/glimpse.hpp index 92eaad47e9..357fbce726 100644 --- a/src/mlpack/methods/ann/layer/glimpse.hpp +++ b/src/mlpack/methods/ann/layer/glimpse.hpp @@ -127,11 +127,6 @@ class Glimpse arma::Mat&& gy, arma::Mat&& g); - //! Get the input parameter. - InputDataType& InputParameter() const {return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const {return outputParameter; } //! Modify the output parameter. @@ -393,9 +388,6 @@ class Glimpse //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/gru.hpp b/src/mlpack/methods/ann/layer/gru.hpp index ed2e96fbcd..63fee576be 100644 --- a/src/mlpack/methods/ann/layer/gru.hpp +++ b/src/mlpack/methods/ann/layer/gru.hpp @@ -135,11 +135,6 @@ class GRU //! Modify the parameters. OutputDataType& Parameters() { return weights; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -246,9 +241,6 @@ class GRU //! Locally-stored gradient object. OutputDataType gradient; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class GRU diff --git a/src/mlpack/methods/ann/layer/hard_tanh.hpp b/src/mlpack/methods/ann/layer/hard_tanh.hpp index 7de2158d6f..8ff75b899f 100644 --- a/src/mlpack/methods/ann/layer/hard_tanh.hpp +++ b/src/mlpack/methods/ann/layer/hard_tanh.hpp @@ -83,11 +83,6 @@ class HardTanH DataType&& gy, DataType&& g); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -118,9 +113,6 @@ class HardTanH //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/join.hpp b/src/mlpack/methods/ann/layer/join.hpp index b55c4bca99..2f6ecde25f 100644 --- a/src/mlpack/methods/ann/layer/join.hpp +++ b/src/mlpack/methods/ann/layer/join.hpp @@ -60,11 +60,6 @@ class Join arma::Mat&& gy, arma::Mat&& g); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -91,9 +86,6 @@ class Join //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class Join diff --git a/src/mlpack/methods/ann/layer/layer_norm.hpp b/src/mlpack/methods/ann/layer/layer_norm.hpp index a978f2b022..2c9d532e23 100644 --- a/src/mlpack/methods/ann/layer/layer_norm.hpp +++ b/src/mlpack/methods/ann/layer/layer_norm.hpp @@ -121,11 +121,6 @@ class LayerNorm //! Modify the parameters. OutputDataType& Parameters() { return weights; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -181,9 +176,6 @@ class LayerNorm //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/leaky_relu.hpp b/src/mlpack/methods/ann/layer/leaky_relu.hpp index dfa7af8e92..70aaf5a231 100644 --- a/src/mlpack/methods/ann/layer/leaky_relu.hpp +++ b/src/mlpack/methods/ann/layer/leaky_relu.hpp @@ -75,11 +75,6 @@ class LeakyReLU template void Backward(const DataType&& input, DataType&& gy, DataType&& g); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -157,9 +152,6 @@ class LeakyReLU //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/log_softmax.hpp b/src/mlpack/methods/ann/layer/log_softmax.hpp index 62654907db..613eac4aa3 100644 --- a/src/mlpack/methods/ann/layer/log_softmax.hpp +++ b/src/mlpack/methods/ann/layer/log_softmax.hpp @@ -65,11 +65,6 @@ class LogSoftMax arma::Mat&& gy, arma::Mat&& g); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -90,9 +85,6 @@ class LogSoftMax //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class LogSoftmax diff --git a/src/mlpack/methods/ann/layer/lookup.hpp b/src/mlpack/methods/ann/layer/lookup.hpp index 5c5109cf0e..59e5547cda 100644 --- a/src/mlpack/methods/ann/layer/lookup.hpp +++ b/src/mlpack/methods/ann/layer/lookup.hpp @@ -85,11 +85,6 @@ class Lookup //! Modify the parameters. OutputDataType& Parameters() { return weights; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -127,9 +122,6 @@ class Lookup //! Locally-stored gradient object. OutputDataType gradient; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class Lookup diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index cd0f867136..2b4deaeb97 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -136,11 +136,6 @@ class LSTM //! Modify the parameters. OutputDataType& Parameters() { return weights; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -206,9 +201,6 @@ class LSTM //! Locally-stored gradient object. OutputDataType grad; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/max_pooling.hpp b/src/mlpack/methods/ann/layer/max_pooling.hpp index 0001819959..762d332d51 100644 --- a/src/mlpack/methods/ann/layer/max_pooling.hpp +++ b/src/mlpack/methods/ann/layer/max_pooling.hpp @@ -94,11 +94,6 @@ class MaxPooling arma::Mat&& gy, arma::Mat&& g); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -253,9 +248,6 @@ class MaxPooling //! Locally-stored gradient object. OutputDataType gradient; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/mean_pooling.hpp b/src/mlpack/methods/ann/layer/mean_pooling.hpp index 3c77d30964..ee4ac585ba 100644 --- a/src/mlpack/methods/ann/layer/mean_pooling.hpp +++ b/src/mlpack/methods/ann/layer/mean_pooling.hpp @@ -74,11 +74,6 @@ class MeanPooling arma::Mat&& gy, arma::Mat&& g); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -234,9 +229,6 @@ class MeanPooling //! Locally-stored gradient object. OutputDataType gradient; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class MeanPooling diff --git a/src/mlpack/methods/ann/layer/multiply_constant.hpp b/src/mlpack/methods/ann/layer/multiply_constant.hpp index c078438889..cdc65659bd 100644 --- a/src/mlpack/methods/ann/layer/multiply_constant.hpp +++ b/src/mlpack/methods/ann/layer/multiply_constant.hpp @@ -60,11 +60,6 @@ class MultiplyConstant template void Backward(const DataType&& /* input */, DataType&& gy, DataType&& g); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -88,9 +83,6 @@ class MultiplyConstant //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class MultiplyConstant diff --git a/src/mlpack/methods/ann/layer/multiply_merge.hpp b/src/mlpack/methods/ann/layer/multiply_merge.hpp index 865f291d76..5b378d23ab 100644 --- a/src/mlpack/methods/ann/layer/multiply_merge.hpp +++ b/src/mlpack/methods/ann/layer/multiply_merge.hpp @@ -99,11 +99,6 @@ class MultiplyMerge template void Add(Args... args) { network.push_back(new LayerType(args...)); } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -156,9 +151,6 @@ class MultiplyMerge //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class MultiplyMerge diff --git a/src/mlpack/methods/ann/layer/negative_log_likelihood.hpp b/src/mlpack/methods/ann/layer/negative_log_likelihood.hpp deleted file mode 100644 index cd92765d1c..0000000000 --- a/src/mlpack/methods/ann/layer/negative_log_likelihood.hpp +++ /dev/null @@ -1,106 +0,0 @@ -/** - * @file negative_log_likelihood.hpp - * @author Marcus Edel - * - * Definition of the NegativeLogLikelihood class. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_NEGATIVE_LOG_LIKELIHOOD_HPP -#define MLPACK_METHODS_ANN_LAYER_NEGATIVE_LOG_LIKELIHOOD_HPP - -#include - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -/** - * Implementation of the negative log likelihood layer. The negative log - * likelihood layer expectes that the input contains log-probabilities for each - * class. The layer also expects a class index, in the range between 1 and the - * number of classes, as target when calling the Forward function. - * - * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, - * arma::sp_mat or arma::cube). - * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, - * arma::sp_mat or arma::cube). - */ -template < - typename InputDataType = arma::mat, - typename OutputDataType = arma::mat -> -class NegativeLogLikelihood -{ - public: - /** - * Create the NegativeLogLikelihoodLayer object. - */ - NegativeLogLikelihood(); - - /* - * Computes the Negative log likelihood. - * - * @param input Input data used for evaluating the specified function. - * @param output Resulting output activation. - */ - template - double Forward(const InputType&& input, TargetType&& target); - - /** - * Ordinary feed backward pass of a neural network. The negative log - * likelihood layer expectes that the input contains log-probabilities for - * each class. The layer also expects a class index, in the range between 1 - * and the number of classes, as target when calling the Forward function. - * - * @param input The propagated input activation. - * @param target The target vector, that contains the class index in the range - * between 1 and the number of classes. - * @param output The calculated error. - */ - template - void Backward(const InputType&& input, - const TargetType&& target, - OutputType&& output); - - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - - //! Get the output parameter. - OutputDataType& OutputParameter() const { return outputParameter; } - //! Modify the output parameter. - OutputDataType& OutputParameter() { return outputParameter; } - - //! Get the delta. - OutputDataType& Delta() const { return delta; } - //! Modify the delta. - OutputDataType& Delta() { return delta; } - - /** - * Serialize the layer - */ - template - void serialize(Archive& /* ar */, const unsigned int /* version */); - - private: - //! Locally-stored delta object. - OutputDataType delta; - - //! Locally-stored input parameter object. - InputDataType inputParameter; - - //! Locally-stored output parameter object. - OutputDataType outputParameter; -}; // class NegativeLogLikelihood - -} // namespace ann -} // namespace mlpack - -// Include implementation. -#include "negative_log_likelihood_impl.hpp" - -#endif diff --git a/src/mlpack/methods/ann/layer/negative_log_likelihood_impl.hpp b/src/mlpack/methods/ann/layer/negative_log_likelihood_impl.hpp deleted file mode 100644 index 922546753b..0000000000 --- a/src/mlpack/methods/ann/layer/negative_log_likelihood_impl.hpp +++ /dev/null @@ -1,75 +0,0 @@ -/** - * @file negative_log_likelihood_impl.hpp - * @author Marcus Edel - * - * Implementation of the NegativeLogLikelihood class. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_NEGATIVE_LOG_LIKELIHOOD_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_NEGATIVE_LOG_LIKELIHOOD_IMPL_HPP - -// In case it hasn't yet been included. -#include "negative_log_likelihood.hpp" - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -template -NegativeLogLikelihood::NegativeLogLikelihood() -{ - // Nothing to do here. -} - -template -template -double NegativeLogLikelihood::Forward( - const InputType&& input, TargetType&& target) -{ - double output = 0; - for (size_t i = 0; i < input.n_cols; ++i) - { - size_t currentTarget = target(i) - 1; - Log::Assert(currentTarget >= 0 && currentTarget < input.n_rows, - "Target class out of range."); - - output -= input(currentTarget, i); - } - - return output; -} - -template -template -void NegativeLogLikelihood::Backward( - const InputType&& input, - const TargetType&& target, - OutputType&& output) -{ - output = arma::zeros(input.n_rows, input.n_cols); - for (size_t i = 0; i < input.n_cols; ++i) - { - size_t currentTarget = target(i) - 1; - Log::Assert(currentTarget >= 0 && currentTarget < input.n_rows, - "Target class out of range."); - - output(currentTarget, i) = -1; - } -} - -template -template -void NegativeLogLikelihood::serialize( - Archive& /* ar */, - const unsigned int /* version */) -{ - // Nothing to do here. -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/parametric_relu.hpp b/src/mlpack/methods/ann/layer/parametric_relu.hpp index 8fcfc7e9c9..79efce5f11 100644 --- a/src/mlpack/methods/ann/layer/parametric_relu.hpp +++ b/src/mlpack/methods/ann/layer/parametric_relu.hpp @@ -99,11 +99,6 @@ class PReLU //! Modify the parameters. OutputDataType& Parameters() { return alpha; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -188,9 +183,6 @@ class PReLU //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/recurrent.hpp b/src/mlpack/methods/ann/layer/recurrent.hpp index e9e4012748..1b48894150 100644 --- a/src/mlpack/methods/ann/layer/recurrent.hpp +++ b/src/mlpack/methods/ann/layer/recurrent.hpp @@ -121,11 +121,6 @@ class Recurrent //! Modify the parameters. OutputDataType& Parameters() { return parameters; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -215,9 +210,6 @@ class Recurrent //! Locally-stored gradient object. OutputDataType gradient; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/recurrent_attention.hpp b/src/mlpack/methods/ann/layer/recurrent_attention.hpp index b42f4d261b..087b6bdb6b 100644 --- a/src/mlpack/methods/ann/layer/recurrent_attention.hpp +++ b/src/mlpack/methods/ann/layer/recurrent_attention.hpp @@ -124,11 +124,6 @@ class RecurrentAttention //! Modify the parameters. OutputDataType& Parameters() { return parameters; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -225,9 +220,6 @@ class RecurrentAttention //! Locally-stored gradient object. OutputDataType gradient; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/reinforce_normal.hpp b/src/mlpack/methods/ann/layer/reinforce_normal.hpp index 25e8c09b99..bb0ea2e737 100644 --- a/src/mlpack/methods/ann/layer/reinforce_normal.hpp +++ b/src/mlpack/methods/ann/layer/reinforce_normal.hpp @@ -63,11 +63,6 @@ class ReinforceNormal template void Backward(const DataType&& input, DataType&& /* gy */, DataType&& g); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -104,9 +99,6 @@ class ReinforceNormal //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/select.hpp b/src/mlpack/methods/ann/layer/select.hpp index cd57d0c27b..4ec71b9e16 100644 --- a/src/mlpack/methods/ann/layer/select.hpp +++ b/src/mlpack/methods/ann/layer/select.hpp @@ -64,11 +64,6 @@ class Select arma::Mat&& gy, arma::Mat&& g); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -95,9 +90,6 @@ class Select //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class Select diff --git a/src/mlpack/methods/ann/layer/sequential.hpp b/src/mlpack/methods/ann/layer/sequential.hpp index b04f8e25cb..0fe120d5a1 100644 --- a/src/mlpack/methods/ann/layer/sequential.hpp +++ b/src/mlpack/methods/ann/layer/sequential.hpp @@ -124,8 +124,9 @@ class Sequential //! Modify the initial point for the optimization. arma::mat& Parameters() { return parameters; } + //! Get the output parameter. arma::mat const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. + //! Modify the output parameter. arma::mat& InputParameter() { return inputParameter; } //! Get the output parameter. diff --git a/src/mlpack/methods/ann/layer/subview.hpp b/src/mlpack/methods/ann/layer/subview.hpp index 735410ef1b..3e20954665 100644 --- a/src/mlpack/methods/ann/layer/subview.hpp +++ b/src/mlpack/methods/ann/layer/subview.hpp @@ -84,11 +84,6 @@ class Subview g = gy; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -119,9 +114,6 @@ class Subview //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class Subview diff --git a/src/mlpack/methods/ann/layer/vr_class_reward.hpp b/src/mlpack/methods/ann/layer/vr_class_reward.hpp index b75e73f436..7a6880f054 100644 --- a/src/mlpack/methods/ann/layer/vr_class_reward.hpp +++ b/src/mlpack/methods/ann/layer/vr_class_reward.hpp @@ -73,11 +73,6 @@ class VRClassReward const TargetType&& target, OutputType&& output); - //! Get the input parameter. - InputDataType& InputParameter() const {return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const {return outputParameter; } //! Modify the output parameter. @@ -127,9 +122,6 @@ class VRClassReward //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt index 786c20bd6d..e372725f95 100644 --- a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt +++ b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt @@ -7,6 +7,8 @@ set(SOURCES kl_divergence_impl.hpp mean_squared_error.hpp mean_squared_error_impl.hpp + negative_log_likelihood.hpp + negative_log_likelihood_impl.hpp sigmoid_cross_entropy_error.hpp sigmoid_cross_entropy_error_impl.hpp ) diff --git a/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp b/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp index dad8eb8c8e..af9e74d26a 100644 --- a/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp @@ -62,21 +62,11 @@ class CrossEntropyError const TargetType&& target, OutputType&& output); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the delta. - OutputDataType& Delta() const { return delta; } - //! Modify the delta. - OutputDataType& Delta() { return delta; } - //! Get the epsilon. double Eps() const { return eps; } //! Modify the epsilon. @@ -89,12 +79,6 @@ class CrossEntropyError void serialize(Archive& ar, const unsigned int /* version */); private: - //! Locally-stored delta object. - OutputDataType delta; - - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp b/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp index 3bfe0b7bff..87f32de6f9 100644 --- a/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp +++ b/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp @@ -74,21 +74,11 @@ class KLDivergence const TargetType&& target, OutputType&& output); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the delta. - OutputDataType& Delta() const { return delta; } - //! Modify the delta. - OutputDataType& Delta() { return delta; } - //! Get the value of takeMean. bool TakeMean() const { return takeMean; } //! Modify the value of takeMean. @@ -101,12 +91,6 @@ class KLDivergence void serialize(Archive& ar, const unsigned int /* version */); private: - //! Locally-stored delta object. - OutputDataType delta; - - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp index 46e829a8ad..59196a23aa 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp @@ -59,21 +59,11 @@ class MeanSquaredError const TargetType&& target, OutputType&& output); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the delta. - OutputDataType& Delta() const { return delta; } - //! Modify the delta. - OutputDataType& Delta() { return delta; } - /** * Serialize the layer */ @@ -81,12 +71,6 @@ class MeanSquaredError void serialize(Archive& ar, const unsigned int /* version */); private: - //! Locally-stored delta object. - OutputDataType delta; - - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class MeanSquaredError diff --git a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp index 91839f69e9..366bf31082 100644 --- a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp @@ -78,21 +78,11 @@ class SigmoidCrossEntropyError const TargetType&& target, OutputType&& output); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the delta. - OutputDataType& Delta() const { return delta; } - //! Modify the delta. - OutputDataType& Delta() { return delta; } - /** * Serialize the layer. */ @@ -100,12 +90,6 @@ class SigmoidCrossEntropyError void serialize(Archive& ar, const unsigned int /* version */); private: - //! Locally-stored delta object. - OutputDataType delta; - - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class SigmoidCrossEntropy From fa3481c02dc1e2f3a1b11cb6504f71c2bf1e026c Mon Sep 17 00:00:00 2001 From: akhandait Date: Sat, 16 Jun 2018 11:58:52 +0530 Subject: [PATCH 50/79] moved NegativeLogLikelihood to loss folder --- .../negative_log_likelihood.hpp | 106 ++++++++++++++++++ .../negative_log_likelihood_impl.hpp | 75 +++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp create mode 100644 src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp new file mode 100644 index 0000000000..cd92765d1c --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp @@ -0,0 +1,106 @@ +/** + * @file negative_log_likelihood.hpp + * @author Marcus Edel + * + * Definition of the NegativeLogLikelihood class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_NEGATIVE_LOG_LIKELIHOOD_HPP +#define MLPACK_METHODS_ANN_LAYER_NEGATIVE_LOG_LIKELIHOOD_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * Implementation of the negative log likelihood layer. The negative log + * likelihood layer expectes that the input contains log-probabilities for each + * class. The layer also expects a class index, in the range between 1 and the + * number of classes, as target when calling the Forward function. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class NegativeLogLikelihood +{ + public: + /** + * Create the NegativeLogLikelihoodLayer object. + */ + NegativeLogLikelihood(); + + /* + * Computes the Negative log likelihood. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + template + double Forward(const InputType&& input, TargetType&& target); + + /** + * Ordinary feed backward pass of a neural network. The negative log + * likelihood layer expectes that the input contains log-probabilities for + * each class. The layer also expects a class index, in the range between 1 + * and the number of classes, as target when calling the Forward function. + * + * @param input The propagated input activation. + * @param target The target vector, that contains the class index in the range + * between 1 and the number of classes. + * @param output The calculated error. + */ + template + void Backward(const InputType&& input, + const TargetType&& target, + OutputType&& output); + + //! Get the input parameter. + InputDataType& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + /** + * Serialize the layer + */ + template + void serialize(Archive& /* ar */, const unsigned int /* version */); + + private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class NegativeLogLikelihood + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "negative_log_likelihood_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp new file mode 100644 index 0000000000..922546753b --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp @@ -0,0 +1,75 @@ +/** + * @file negative_log_likelihood_impl.hpp + * @author Marcus Edel + * + * Implementation of the NegativeLogLikelihood class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_NEGATIVE_LOG_LIKELIHOOD_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_NEGATIVE_LOG_LIKELIHOOD_IMPL_HPP + +// In case it hasn't yet been included. +#include "negative_log_likelihood.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +NegativeLogLikelihood::NegativeLogLikelihood() +{ + // Nothing to do here. +} + +template +template +double NegativeLogLikelihood::Forward( + const InputType&& input, TargetType&& target) +{ + double output = 0; + for (size_t i = 0; i < input.n_cols; ++i) + { + size_t currentTarget = target(i) - 1; + Log::Assert(currentTarget >= 0 && currentTarget < input.n_rows, + "Target class out of range."); + + output -= input(currentTarget, i); + } + + return output; +} + +template +template +void NegativeLogLikelihood::Backward( + const InputType&& input, + const TargetType&& target, + OutputType&& output) +{ + output = arma::zeros(input.n_rows, input.n_cols); + for (size_t i = 0; i < input.n_cols; ++i) + { + size_t currentTarget = target(i) - 1; + Log::Assert(currentTarget >= 0 && currentTarget < input.n_rows, + "Target class out of range."); + + output(currentTarget, i) = -1; + } +} + +template +template +void NegativeLogLikelihood::serialize( + Archive& /* ar */, + const unsigned int /* version */) +{ + // Nothing to do here. +} + +} // namespace ann +} // namespace mlpack + +#endif From 51d4306c51d31e1164c0174fc58ce70705ceb78e Mon Sep 17 00:00:00 2001 From: akhandait Date: Sat, 16 Jun 2018 12:06:38 +0530 Subject: [PATCH 51/79] rectified mistake in last commit, added comment --- src/mlpack/methods/ann/layer/sequential.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/layer/sequential.hpp b/src/mlpack/methods/ann/layer/sequential.hpp index 0fe120d5a1..42a82f8d76 100644 --- a/src/mlpack/methods/ann/layer/sequential.hpp +++ b/src/mlpack/methods/ann/layer/sequential.hpp @@ -124,9 +124,9 @@ class Sequential //! Modify the initial point for the optimization. arma::mat& Parameters() { return parameters; } - //! Get the output parameter. + //! Get the input parameter. arma::mat const& InputParameter() const { return inputParameter; } - //! Modify the output parameter. + //! Modify the input parameter. arma::mat& InputParameter() { return inputParameter; } //! Get the output parameter. From dae8194eba81c26d1ec743b8c537802520a0dc69 Mon Sep 17 00:00:00 2001 From: akhandait Date: Sat, 16 Jun 2018 22:59:24 +0530 Subject: [PATCH 52/79] changed path --- src/mlpack/methods/ann/layer/layer_types.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 3584aeb5bb..fb27a44da8 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -44,6 +43,9 @@ #include #include +// Loss function modules. +#include + namespace mlpack { namespace ann { From 85e1c3bcbf12f66466efe098390cc7383cca794d Mon Sep 17 00:00:00 2001 From: akhandait Date: Fri, 1 Jun 2018 11:27:00 +0530 Subject: [PATCH 53/79] added sampling layer --- src/mlpack/methods/ann/layer/CMakeLists.txt | 2 + src/mlpack/methods/ann/layer/layer.hpp | 1 + src/mlpack/methods/ann/layer/layer_types.hpp | 3 + src/mlpack/methods/ann/layer/linear.hpp | 2 +- src/mlpack/methods/ann/layer/sampling.hpp | 177 ++++++++++++++++++ .../methods/ann/layer/sampling_impl.hpp | 110 +++++++++++ 6 files changed, 294 insertions(+), 1 deletion(-) create mode 100644 src/mlpack/methods/ann/layer/sampling.hpp create mode 100644 src/mlpack/methods/ann/layer/sampling_impl.hpp diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt index 6523499f74..8d76577727 100644 --- a/src/mlpack/methods/ann/layer/CMakeLists.txt +++ b/src/mlpack/methods/ann/layer/CMakeLists.txt @@ -75,6 +75,8 @@ set(SOURCES recurrent_attention_impl.hpp reinforce_normal.hpp reinforce_normal_impl.hpp + sampling.hpp + sampling_impl.hpp select.hpp select_impl.hpp sequential.hpp diff --git a/src/mlpack/methods/ann/layer/layer.hpp b/src/mlpack/methods/ann/layer/layer.hpp index e3395a1586..7eb03f7dfc 100644 --- a/src/mlpack/methods/ann/layer/layer.hpp +++ b/src/mlpack/methods/ann/layer/layer.hpp @@ -29,6 +29,7 @@ #include "fast_lstm.hpp" #include "recurrent.hpp" #include "recurrent_attention.hpp" +#include "sampling.hpp" #include "sequential.hpp" #include "subview.hpp" #include "concat.hpp" diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 3584aeb5bb..bd67c25084 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -56,6 +57,7 @@ template class LinearNoBias; template class LSTM; template class GRU; template class FastLSTM; +template class Sampling; template class VRClassReward; template*, RecurrentAttention*, ReinforceNormal*, + Sampling*, Select*, Sequential*, Subview*, diff --git a/src/mlpack/methods/ann/layer/linear.hpp b/src/mlpack/methods/ann/layer/linear.hpp index 7de48ece1b..3757533363 100644 --- a/src/mlpack/methods/ann/layer/linear.hpp +++ b/src/mlpack/methods/ann/layer/linear.hpp @@ -129,7 +129,7 @@ class Linear //! Locally-stored weight object. OutputDataType weights; - //! Locally-stored weight paramters. + //! Locally-stored weight parameters. OutputDataType weight; //! Locally-stored bias term parameters. diff --git a/src/mlpack/methods/ann/layer/sampling.hpp b/src/mlpack/methods/ann/layer/sampling.hpp new file mode 100644 index 0000000000..b2e00e2819 --- /dev/null +++ b/src/mlpack/methods/ann/layer/sampling.hpp @@ -0,0 +1,177 @@ +/** + * @file sampling.hpp + * @author Atharva Khandait + * + * Definition of the Sampling layer class which samples from parameters for a given + * distribution. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_SAMPLING_HPP +#define MLPACK_METHODS_ANN_LAYER_SAMPLING_HPP + +#include + +#include "layer_types.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * Implementation of the Sampling layer class. This layer samples from the given + * parameters of a normal distribution. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class Sampling +{ + public: + //! Create the Sampling object. + Sampling(); + + /** + * Create the Sampling layer object using the specified number of units. + * + * @param inSize The number of input units. + * @param outSize The number of output units. + */ + Sampling(const size_t inSize, const size_t outSize); + + /** + * Create the Sampling layer object using the specified sample vector size. + * + * @param layerSize The number of output units. + */ + Sampling(const size_t sampleSize); + + /* + * Reset the layer parameter. + */ + void Reset(); + + /** + * Ordinary feed forward pass of a neural network, evaluating the function + * f(x) by propagating the activity forward through f. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + template + void Forward(const arma::Mat&& input, arma::Mat&& output); + + /** + * Ordinary feed backward pass of a neural network, calculating the function + * f(x) by propagating x backwards trough f. Using the results from the feed + * forward pass. + * + * @param input The propagated input activation. + * @param gy The backpropagated error. + * @param g The calculated gradient. + */ + template + void Backward(const arma::Mat&& /* input */, + arma::Mat&& gy, + arma::Mat&& g); + + /* + * Calculate the gradient using the output delta and the input activation. + * + * @param input The input parameter used for calculating the gradient. + * @param error The calculated error. + * @param gradient The calculated gradient. + */ + template + void Gradient(const arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& gradient); + + //! Get the parameters. + OutputDataType const& Parameters() const { return weights; } + //! Modify the parameters. + OutputDataType& Parameters() { return weights; } + + //! Get the input parameter. + InputDataType const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } + + //! Get the input size. + size_t const& InputSize() const { return inSize; } + //! Modify the input size. + size_t& InputSize() { return inSize; } + + //! Get the output size. + size_t const& OutputSize() const { return outSize; } + //! Modify the output size. + size_t& OutputSize() { return outSize; } + + /** + * Serialize the layer + */ + template + void serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Locally-stored number of input units. + size_t inSize; + + //! Locally-stored number of output units. + size_t outSize; + + //! Locally-stored weight object. + OutputDataType weights; + + //! Locally-stored weight parameters. + OutputDataType weight; + + //! Locally-stored bias term parameters. + OutputDataType bias; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored current gaussian sample. + OutputDataType gaussianSample; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class Sampling + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "sampling_impl.hpp" + +#endif \ No newline at end of file diff --git a/src/mlpack/methods/ann/layer/sampling_impl.hpp b/src/mlpack/methods/ann/layer/sampling_impl.hpp new file mode 100644 index 0000000000..4215662047 --- /dev/null +++ b/src/mlpack/methods/ann/layer/sampling_impl.hpp @@ -0,0 +1,110 @@ +/** + * @file sampling_impl.hpp + * @author Atharva Khandait + * + * Implementation of the Sampling class which samples from parameters for a given + * distribution. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_SAMPLING_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_SAMPLING_IMPL_HPP + +// In case it hasn't yet been included. +#include "sampling.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +Sampling::Sampling() +{ + // Nothing to do here. +} + +template +Sampling::Sampling( + const size_t inSize, + const size_t outSize) : + inSize(inSize), + outSize(outSize) +{ + weights.set_size(2 * outSize * inSize + 2 * outSize, 1); +} + +template +Sampling::Sampling( + const size_t sampleSize) : + outSize(sampleSize) +{ + // Nothing to do here. +} + +template +void Sampling::Reset() +{ + weights.set_size(2 * outSize * inSize + 2 * outSize, 1); + + weight = arma::mat(weights.memptr(), 2 * outSize, inSize, false, false); + bias = arma::mat(weights.memptr() + weight.n_elem, + 2 * outSize, 1, false, false); +} + +template +template +void Sampling::Forward( + const arma::Mat&& input, arma::Mat&& output) +{ + arma::arma_rng::set_seed_random(); + + output = weight * input; + output.each_col() += bias; + gaussianSample = arma::randn(outSize, output.n_cols); + output = (output.submat(outSize, 0, 2 * outSize - 1, output.n_cols - 1) + + output.submat(0, 0, outSize - 1, output.n_cols - 1)) % gaussianSample; +} + +template +template +void Sampling::Backward( + const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) +{std::cout << gy.n_rows << std::endl << gy.n_cols << std::endl; + g = join_cols((weight.submat(0, 0, outSize - 1, inSize - 1).t() * gy) % + gaussianSample, + weight.submat(outSize, 0, 2 * outSize - 1, inSize - 1).t() * gy); +} + +template +template +void Sampling::Gradient( + const arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& gradient) +{ + gradient.submat(0, 0, weight.n_elem - 1, 0) = arma::vectorise( + error * input.t()); + gradient.submat(weight.n_elem, 0, gradient.n_elem - 1, 0) = + arma::sum(error, 1); +} + +template +template +void Sampling::serialize( + Archive& ar, const unsigned int /* version */) +{ + ar & BOOST_SERIALIZATION_NVP(inSize); + ar & BOOST_SERIALIZATION_NVP(outSize); + + // This is inefficient, but we have to allocate this memory so that + // WeightSetVisitor gets the right size. + if (Archive::is_loading::value) + weights.set_size(2 * outSize * inSize + 2 * outSize, 1); +} + +} // namespace ann +} // namespace mlpack + +#endif \ No newline at end of file From 67ab902fd01bdd2731ff5bd14167218628160feb Mon Sep 17 00:00:00 2001 From: akhandait Date: Fri, 1 Jun 2018 17:20:56 +0530 Subject: [PATCH 54/79] removed parameters, it just does the reparametrization now --- src/mlpack/methods/ann/layer/sampling.hpp | 4 +- .../methods/ann/layer/sampling_impl.hpp | 44 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/mlpack/methods/ann/layer/sampling.hpp b/src/mlpack/methods/ann/layer/sampling.hpp index b2e00e2819..9beec42f6e 100644 --- a/src/mlpack/methods/ann/layer/sampling.hpp +++ b/src/mlpack/methods/ann/layer/sampling.hpp @@ -45,7 +45,7 @@ class Sampling * @param inSize The number of input units. * @param outSize The number of output units. */ - Sampling(const size_t inSize, const size_t outSize); + // Sampling(const size_t inSize, const size_t outSize); /** * Create the Sampling layer object using the specified sample vector size. @@ -57,7 +57,7 @@ class Sampling /* * Reset the layer parameter. */ - void Reset(); + // void Reset(); /** * Ordinary feed forward pass of a neural network, evaluating the function diff --git a/src/mlpack/methods/ann/layer/sampling_impl.hpp b/src/mlpack/methods/ann/layer/sampling_impl.hpp index 4215662047..48be2a8c4f 100644 --- a/src/mlpack/methods/ann/layer/sampling_impl.hpp +++ b/src/mlpack/methods/ann/layer/sampling_impl.hpp @@ -25,15 +25,15 @@ Sampling::Sampling() // Nothing to do here. } -template -Sampling::Sampling( - const size_t inSize, - const size_t outSize) : - inSize(inSize), - outSize(outSize) -{ - weights.set_size(2 * outSize * inSize + 2 * outSize, 1); -} +// template +// Sampling::Sampling( +// const size_t inSize, +// const size_t outSize) : +// inSize(inSize), +// outSize(outSize) +// { +// weights.set_size(2 * outSize * inSize + 2 * outSize, 1); +// } template Sampling::Sampling( @@ -43,15 +43,15 @@ Sampling::Sampling( // Nothing to do here. } -template -void Sampling::Reset() -{ - weights.set_size(2 * outSize * inSize + 2 * outSize, 1); +// template +// void Sampling::Reset() +// { +// weights.set_size(2 * outSize * inSize + 2 * outSize, 1); - weight = arma::mat(weights.memptr(), 2 * outSize, inSize, false, false); - bias = arma::mat(weights.memptr() + weight.n_elem, - 2 * outSize, 1, false, false); -} +// weight = arma::mat(weights.memptr(), 2 * outSize, inSize, false, false); +// bias = arma::mat(weights.memptr() + weight.n_elem, +// 2 * outSize, 1, false, false); +// } template template @@ -60,11 +60,11 @@ void Sampling::Forward( { arma::arma_rng::set_seed_random(); - output = weight * input; - output.each_col() += bias; - gaussianSample = arma::randn(outSize, output.n_cols); - output = (output.submat(outSize, 0, 2 * outSize - 1, output.n_cols - 1) + - output.submat(0, 0, outSize - 1, output.n_cols - 1)) % gaussianSample; + // output = weight * input; + // output.each_col() += bias; + gaussianSample = arma::randn(outSize, input.n_cols); + output = (input.submat(outSize, 0, 2 * outSize - 1, input.n_cols - 1) + + input.submat(0, 0, outSize - 1, input.n_cols - 1)) % gaussianSample; } template From 3a7bb6a3f69d4bd180df6c9bc37c567341dfd566 Mon Sep 17 00:00:00 2001 From: akhandait Date: Mon, 4 Jun 2018 12:54:32 +0530 Subject: [PATCH 55/79] sampling layer done, kl divergence forward implemented --- src/mlpack/methods/ann/layer/sampling.hpp | 72 ++++++++--------- .../methods/ann/layer/sampling_impl.hpp | 81 +++++++++---------- src/mlpack/tests/ann_layer_test.cpp | 17 ++++ 3 files changed, 86 insertions(+), 84 deletions(-) diff --git a/src/mlpack/methods/ann/layer/sampling.hpp b/src/mlpack/methods/ann/layer/sampling.hpp index 9beec42f6e..61df2c7da7 100644 --- a/src/mlpack/methods/ann/layer/sampling.hpp +++ b/src/mlpack/methods/ann/layer/sampling.hpp @@ -2,7 +2,7 @@ * @file sampling.hpp * @author Atharva Khandait * - * Definition of the Sampling layer class which samples from parameters for a given + * Definition of the Sampling layer class which samples from a gaussian * distribution. * * mlpack is free software; you may redistribute it and/or modify it under the @@ -16,13 +16,14 @@ #include #include "layer_types.hpp" +#include "../activation_functions/softplus_function.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * Implementation of the Sampling layer class. This layer samples from the given - * parameters of a normal distribution. + * Implementation of the Sampling layer class. This layer samples from the + * given parameters of a normal distribution. * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). @@ -45,7 +46,7 @@ class Sampling * @param inSize The number of input units. * @param outSize The number of output units. */ - // Sampling(const size_t inSize, const size_t outSize); + Sampling(const size_t inSize, const size_t outSize); /** * Create the Sampling layer object using the specified sample vector size. @@ -54,11 +55,6 @@ class Sampling */ Sampling(const size_t sampleSize); - /* - * Reset the layer parameter. - */ - // void Reset(); - /** * Ordinary feed forward pass of a neural network, evaluating the function * f(x) by propagating the activity forward through f. @@ -79,26 +75,33 @@ class Sampling * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, + void Backward(const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g); - /* - * Calculate the gradient using the output delta and the input activation. + /** + * Ordinary feed forward pass of a neural network, evaluating + * Kullback–Leibler divergence between a normal distribution + * and the standard normal. * - * @param input The input parameter used for calculating the gradient. - * @param error The calculated error. - * @param gradient The calculated gradient. + * @param input Input data used for evaluating the specified function. + */ + template + double klForward(); + + /** + * Ordinary feed backward pass of a neural network, evaluating the backward + * pass of Kullback–Leibler divergence. Using the results from the + * KL divergence feed forward pass. + * + * @param input The propagated input activation. + * @param gy The backpropagated error. + * @param g The calculated gradient. */ template - void Gradient(const arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient); - - //! Get the parameters. - OutputDataType const& Parameters() const { return weights; } - //! Modify the parameters. - OutputDataType& Parameters() { return weights; } + void klBackward(const arma::Mat&& input, + arma::Mat&& gy, + arma::Mat&& g); //! Get the input parameter. InputDataType const& InputParameter() const { return inputParameter; } @@ -115,11 +118,6 @@ class Sampling //! Modify the delta. OutputDataType& Delta() { return delta; } - //! Get the gradient. - OutputDataType const& Gradient() const { return gradient; } - //! Modify the gradient. - OutputDataType& Gradient() { return gradient; } - //! Get the input size. size_t const& InputSize() const { return inSize; } //! Modify the input size. @@ -143,24 +141,18 @@ class Sampling //! Locally-stored number of output units. size_t outSize; - //! Locally-stored weight object. - OutputDataType weights; - - //! Locally-stored weight parameters. - OutputDataType weight; - - //! Locally-stored bias term parameters. - OutputDataType bias; - //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored gradient object. - OutputDataType gradient; - //! Locally-stored current gaussian sample. OutputDataType gaussianSample; + //! Locally-stored current mean. + OutputDataType mean; + + //! Locally-stored current standard deviation. + OutputDataType stdDeviation; + //! Locally-stored input parameter object. InputDataType inputParameter; diff --git a/src/mlpack/methods/ann/layer/sampling_impl.hpp b/src/mlpack/methods/ann/layer/sampling_impl.hpp index 48be2a8c4f..3d1b11dec0 100644 --- a/src/mlpack/methods/ann/layer/sampling_impl.hpp +++ b/src/mlpack/methods/ann/layer/sampling_impl.hpp @@ -25,15 +25,19 @@ Sampling::Sampling() // Nothing to do here. } -// template -// Sampling::Sampling( -// const size_t inSize, -// const size_t outSize) : -// inSize(inSize), -// outSize(outSize) -// { -// weights.set_size(2 * outSize * inSize + 2 * outSize, 1); -// } +template +Sampling::Sampling( + const size_t inSize, + const size_t outSize) : + inSize(inSize), + outSize(outSize) +{ + if (inSize != 2 * outSize) + { + Log::Fatal << "The input size of Sampling layer should be 2 * output size!" + << std::endl; + } +} template Sampling::Sampling( @@ -43,51 +47,45 @@ Sampling::Sampling( // Nothing to do here. } -// template -// void Sampling::Reset() -// { -// weights.set_size(2 * outSize * inSize + 2 * outSize, 1); - -// weight = arma::mat(weights.memptr(), 2 * outSize, inSize, false, false); -// bias = arma::mat(weights.memptr() + weight.n_elem, -// 2 * outSize, 1, false, false); -// } - template template void Sampling::Forward( const arma::Mat&& input, arma::Mat&& output) { + if (input.n_rows != 2 * outSize) + { + Log::Fatal << "The output size of layer before the Sampling layer should " + << "be 2 * output size of the Sampling layer!" << std::endl; + } + arma::arma_rng::set_seed_random(); - // output = weight * input; - // output.each_col() += bias; - gaussianSample = arma::randn(outSize, input.n_cols); - output = (input.submat(outSize, 0, 2 * outSize - 1, input.n_cols - 1) + - input.submat(0, 0, outSize - 1, input.n_cols - 1)) % gaussianSample; + mean = input.submat(outSize, 0, 2 * outSize - 1, input.n_cols - 1); + SoftplusFunction::Fn(input.submat(0, 0, outSize - 1, input.n_cols - 1), + stdDeviation); + + gaussianSample = arma::randn>(outSize, input.n_cols); + output = mean + stdDeviation % gaussianSample; } template template void Sampling::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) -{std::cout << gy.n_rows << std::endl << gy.n_cols << std::endl; - g = join_cols((weight.submat(0, 0, outSize - 1, inSize - 1).t() * gy) % - gaussianSample, - weight.submat(outSize, 0, 2 * outSize - 1, inSize - 1).t() * gy); + const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) +{ + arma::Mat softplusDer; + SoftplusFunction::Deriv((input - mean) / gaussianSample, + softplusDer); + + g = join_cols(gy % std::move(gaussianSample) % std::move(softplusDer), gy); } template -template -void Sampling::Gradient( - const arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient) +template +double Sampling::klForward() { - gradient.submat(0, 0, weight.n_elem - 1, 0) = arma::vectorise( - error * input.t()); - gradient.submat(weight.n_elem, 0, gradient.n_elem - 1, 0) = - arma::sum(error, 1); + return -0.5 * arma::accu(arma::log(stdDeviation) - stdDeviation - + arma::pow(mean, 2) + 1); } template @@ -95,13 +93,8 @@ template void Sampling::serialize( Archive& ar, const unsigned int /* version */) { - ar & BOOST_SERIALIZATION_NVP(inSize); + // ar & BOOST_SERIALIZATION_NVP(inSize); ar & BOOST_SERIALIZATION_NVP(outSize); - - // This is inefficient, but we have to allocate this memory so that - // WeightSetVisitor gets the right size. - if (Archive::is_loading::value) - weights.set_size(2 * outSize * inSize + 2 * outSize, 1); } } // namespace ann diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index aaae439aec..04d26600cc 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -1847,4 +1847,21 @@ BOOST_AUTO_TEST_CASE(SubviewIndexTest) CheckMatrices(outputEnd, subEnd); } +/* + * Simple Sampling module test. + */ +BOOST_AUTO_TEST_CASE(SimpleSamplingLayerTest) +{ + arma::mat input, output, delta; + Sampling<> module(10, 5); + + // Test the Forward function. + input = join_cols(arma::ones(5, 1) * -10, + arma::zeros(5, 1)); + module.Forward(std::move(input), std::move(output)); + BOOST_REQUIRE_LE(arma::accu(output), 1e-3); + + // Test the Backward function. +} + BOOST_AUTO_TEST_SUITE_END(); From c0750ff8ac24a297c306104e7e465094f06eb2e2 Mon Sep 17 00:00:00 2001 From: akhandait Date: Mon, 4 Jun 2018 16:50:42 +0530 Subject: [PATCH 56/79] kl backward implemented --- src/mlpack/methods/ann/layer/sampling.hpp | 9 +++---- .../methods/ann/layer/sampling_impl.hpp | 25 ++++++++++++++----- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/mlpack/methods/ann/layer/sampling.hpp b/src/mlpack/methods/ann/layer/sampling.hpp index 61df2c7da7..39e4d91ff3 100644 --- a/src/mlpack/methods/ann/layer/sampling.hpp +++ b/src/mlpack/methods/ann/layer/sampling.hpp @@ -87,7 +87,7 @@ class Sampling * @param input Input data used for evaluating the specified function. */ template - double klForward(); + double klForward(const InputType&& input); /** * Ordinary feed backward pass of a neural network, evaluating the backward @@ -98,10 +98,9 @@ class Sampling * @param gy The backpropagated error. * @param g The calculated gradient. */ - template - void klBackward(const arma::Mat&& input, - arma::Mat&& gy, - arma::Mat&& g); + template + void klBackward(const InputType&& input, + OutputType&& output); //! Get the input parameter. InputDataType const& InputParameter() const { return inputParameter; } diff --git a/src/mlpack/methods/ann/layer/sampling_impl.hpp b/src/mlpack/methods/ann/layer/sampling_impl.hpp index 3d1b11dec0..d383cf36ca 100644 --- a/src/mlpack/methods/ann/layer/sampling_impl.hpp +++ b/src/mlpack/methods/ann/layer/sampling_impl.hpp @@ -65,16 +65,16 @@ void Sampling::Forward( stdDeviation); gaussianSample = arma::randn>(outSize, input.n_cols); - output = mean + stdDeviation % gaussianSample; + output = mean + std::move(stdDeviation) % gaussianSample; } template template void Sampling::Backward( const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) -{ +{ arma::Mat softplusDer; - SoftplusFunction::Deriv((input - mean) / gaussianSample, + SoftplusFunction::Deriv((input - std::move(mean)) / gaussianSample, softplusDer); g = join_cols(gy % std::move(gaussianSample) % std::move(softplusDer), gy); @@ -82,10 +82,23 @@ void Sampling::Backward( template template -double Sampling::klForward() +double Sampling::klForward( + const InputType&& input) { - return -0.5 * arma::accu(arma::log(stdDeviation) - stdDeviation - - arma::pow(mean, 2) + 1); + stdDeviation = input.submat(0, 0, outSize - 1, input.n_cols); + + return -0.5 * arma::accu(arma::log(stdDeviation) - stdDeviation - arma::pow( + input.submat(outSize, 0, 2 * outSize - 1, input.n_cols), 2) + 1); +} + +template +template +void Sampling::klBackward( + const InputType&& input, + OutputType&& output) +{ + output = join_cols(-1 / input.submat(0, 0, outSize - 1, input.n_cols) - 1, + input.submat(outSize, 0, 2 * outSize - 1, input.n_cols)) } template From e61d02139dc637ef5b6d160bd83afdbd62ab1baf Mon Sep 17 00:00:00 2001 From: akhandait Date: Mon, 4 Jun 2018 17:00:24 +0530 Subject: [PATCH 57/79] fix style errors --- src/mlpack/methods/ann/layer/sampling.hpp | 2 +- src/mlpack/methods/ann/layer/sampling_impl.hpp | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/mlpack/methods/ann/layer/sampling.hpp b/src/mlpack/methods/ann/layer/sampling.hpp index 39e4d91ff3..2707215810 100644 --- a/src/mlpack/methods/ann/layer/sampling.hpp +++ b/src/mlpack/methods/ann/layer/sampling.hpp @@ -165,4 +165,4 @@ class Sampling // Include implementation. #include "sampling_impl.hpp" -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/methods/ann/layer/sampling_impl.hpp b/src/mlpack/methods/ann/layer/sampling_impl.hpp index d383cf36ca..a42179c1c9 100644 --- a/src/mlpack/methods/ann/layer/sampling_impl.hpp +++ b/src/mlpack/methods/ann/layer/sampling_impl.hpp @@ -34,8 +34,8 @@ Sampling::Sampling( { if (inSize != 2 * outSize) { - Log::Fatal << "The input size of Sampling layer should be 2 * output size!" - << std::endl; + Log::Fatal << "The input size of Sampling layer should be 2 * output size!" + << std::endl; } } @@ -54,8 +54,8 @@ void Sampling::Forward( { if (input.n_rows != 2 * outSize) { - Log::Fatal << "The output size of layer before the Sampling layer should " - << "be 2 * output size of the Sampling layer!" << std::endl; + Log::Fatal << "The output size of layer before the Sampling layer should " + << "be 2 * output size of the Sampling layer!" << std::endl; } arma::arma_rng::set_seed_random(); @@ -113,4 +113,4 @@ void Sampling::serialize( } // namespace ann } // namespace mlpack -#endif \ No newline at end of file +#endif From 1cf78f12396e90dfe51905c9b0ce2e272ac1073a Mon Sep 17 00:00:00 2001 From: akhandait Date: Tue, 5 Jun 2018 01:01:25 +0530 Subject: [PATCH 58/79] suggested changes made --- .../{sampling.hpp => reparametrization.hpp} | 37 ++++------- ...ng_impl.hpp => reparametrization_impl.hpp} | 65 +++++++------------ 2 files changed, 38 insertions(+), 64 deletions(-) rename src/mlpack/methods/ann/layer/{sampling.hpp => reparametrization.hpp} (83%) rename src/mlpack/methods/ann/layer/{sampling_impl.hpp => reparametrization_impl.hpp} (50%) diff --git a/src/mlpack/methods/ann/layer/sampling.hpp b/src/mlpack/methods/ann/layer/reparametrization.hpp similarity index 83% rename from src/mlpack/methods/ann/layer/sampling.hpp rename to src/mlpack/methods/ann/layer/reparametrization.hpp index 2707215810..3f79906e09 100644 --- a/src/mlpack/methods/ann/layer/sampling.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization.hpp @@ -1,8 +1,8 @@ /** - * @file sampling.hpp + * @file reparametrization.hpp * @author Atharva Khandait * - * Definition of the Sampling layer class which samples from a gaussian + * Definition of the Reparametrization layer class which samples from a gaussian * distribution. * * mlpack is free software; you may redistribute it and/or modify it under the @@ -10,8 +10,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_METHODS_ANN_LAYER_SAMPLING_HPP -#define MLPACK_METHODS_ANN_LAYER_SAMPLING_HPP +#ifndef MLPACK_METHODS_ANN_LAYER_REPARAMETRIZATION_HPP +#define MLPACK_METHODS_ANN_LAYER_REPARAMETRIZATION_HPP #include @@ -22,7 +22,7 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * Implementation of the Sampling layer class. This layer samples from the + * Implementation of the Reparametrization layer class. This layer samples from the * given parameters of a normal distribution. * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, @@ -34,26 +34,18 @@ template < typename InputDataType = arma::mat, typename OutputDataType = arma::mat > -class Sampling +class Reparametrization { public: - //! Create the Sampling object. - Sampling(); + //! Create the Reparametrization object. + Reparametrization(); /** - * Create the Sampling layer object using the specified number of units. - * - * @param inSize The number of input units. - * @param outSize The number of output units. - */ - Sampling(const size_t inSize, const size_t outSize); - - /** - * Create the Sampling layer object using the specified sample vector size. + * Create the Reparametrization layer object using the specified sample vector size. * * @param layerSize The number of output units. */ - Sampling(const size_t sampleSize); + Reparametrization(const size_t latentSize); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -134,11 +126,8 @@ class Sampling void serialize(Archive& ar, const unsigned int /* version */); private: - //! Locally-stored number of input units. - size_t inSize; - //! Locally-stored number of output units. - size_t outSize; + size_t latentSize; //! Locally-stored delta object. OutputDataType delta; @@ -157,12 +146,12 @@ class Sampling //! Locally-stored output parameter object. OutputDataType outputParameter; -}; // class Sampling +}; // class Reparametrization } // namespace ann } // namespace mlpack // Include implementation. -#include "sampling_impl.hpp" +#include "reparametrization_impl.hpp" #endif diff --git a/src/mlpack/methods/ann/layer/sampling_impl.hpp b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp similarity index 50% rename from src/mlpack/methods/ann/layer/sampling_impl.hpp rename to src/mlpack/methods/ann/layer/reparametrization_impl.hpp index a42179c1c9..b40daa8fbe 100644 --- a/src/mlpack/methods/ann/layer/sampling_impl.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp @@ -1,8 +1,8 @@ /** - * @file sampling_impl.hpp + * @file reparametrization_impl.hpp * @author Atharva Khandait * - * Implementation of the Sampling class which samples from parameters for a given + * Implementation of the Reparametrization class which samples from parameters for a given * distribution. * * mlpack is free software; you may redistribute it and/or modify it under the @@ -10,67 +10,53 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_METHODS_ANN_LAYER_SAMPLING_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_SAMPLING_IMPL_HPP +#ifndef MLPACK_METHODS_ANN_LAYER_REPARAMETRIZATION_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_REPARAMETRIZATION_IMPL_HPP // In case it hasn't yet been included. -#include "sampling.hpp" +#include "reparametrization.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -Sampling::Sampling() +Reparametrization::Reparametrization() { // Nothing to do here. } template -Sampling::Sampling( - const size_t inSize, - const size_t outSize) : - inSize(inSize), - outSize(outSize) -{ - if (inSize != 2 * outSize) - { - Log::Fatal << "The input size of Sampling layer should be 2 * output size!" - << std::endl; - } -} - -template -Sampling::Sampling( - const size_t sampleSize) : - outSize(sampleSize) +Reparametrization::Reparametrization( + const size_t latentSize) : + latentSize(latentSize) { // Nothing to do here. } template template -void Sampling::Forward( +void Reparametrization::Forward( const arma::Mat&& input, arma::Mat&& output) { - if (input.n_rows != 2 * outSize) + if (input.n_rows != 2 * latentSize) { - Log::Fatal << "The output size of layer before the Sampling layer should " - << "be 2 * output size of the Sampling layer!" << std::endl; + Log::Fatal << "The output size of layer before the Reparametrization layer should " + << "be 2 * latent size of the Reparametrization layer!" << std::endl; } arma::arma_rng::set_seed_random(); - mean = input.submat(outSize, 0, 2 * outSize - 1, input.n_cols - 1); - SoftplusFunction::Fn(input.submat(0, 0, outSize - 1, input.n_cols - 1), + mean = input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols - 1); + SoftplusFunction::Fn(input.submat(0, 0, latentSize - 1, input.n_cols - 1), stdDeviation); - gaussianSample = arma::randn>(outSize, input.n_cols); + gaussianSample = arma::randn>(latentSize, input.n_cols); output = mean + std::move(stdDeviation) % gaussianSample; } template template -void Sampling::Backward( +void Reparametrization::Backward( const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) { arma::Mat softplusDer; @@ -82,32 +68,31 @@ void Sampling::Backward( template template -double Sampling::klForward( +double Reparametrization::klForward( const InputType&& input) { - stdDeviation = input.submat(0, 0, outSize - 1, input.n_cols); + stdDeviation = input.submat(0, 0, latentSize - 1, input.n_cols); return -0.5 * arma::accu(arma::log(stdDeviation) - stdDeviation - arma::pow( - input.submat(outSize, 0, 2 * outSize - 1, input.n_cols), 2) + 1); + input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols), 2) + 1); } template template -void Sampling::klBackward( +void Reparametrization::klBackward( const InputType&& input, OutputType&& output) { - output = join_cols(-1 / input.submat(0, 0, outSize - 1, input.n_cols) - 1, - input.submat(outSize, 0, 2 * outSize - 1, input.n_cols)) + output = join_cols(-1 / input.submat(0, 0, latentSize - 1, input.n_cols) - 1, + input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols)) } template template -void Sampling::serialize( +void Reparametrization::serialize( Archive& ar, const unsigned int /* version */) { - // ar & BOOST_SERIALIZATION_NVP(inSize); - ar & BOOST_SERIALIZATION_NVP(outSize); + ar & BOOST_SERIALIZATION_NVP(latentSize); } } // namespace ann From 8621c8a15ebf3388e9e16c0377edc1ecac797ffa Mon Sep 17 00:00:00 2001 From: akhandait Date: Tue, 5 Jun 2018 01:03:21 +0530 Subject: [PATCH 59/79] seed removed --- src/mlpack/methods/ann/layer/reparametrization_impl.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/mlpack/methods/ann/layer/reparametrization_impl.hpp b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp index b40daa8fbe..7e5cb87645 100644 --- a/src/mlpack/methods/ann/layer/reparametrization_impl.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp @@ -44,8 +44,6 @@ void Reparametrization::Forward( << "be 2 * latent size of the Reparametrization layer!" << std::endl; } - arma::arma_rng::set_seed_random(); - mean = input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols - 1); SoftplusFunction::Fn(input.submat(0, 0, latentSize - 1, input.n_cols - 1), stdDeviation); From c1d3eec9e8ee467b4117a595c1bbd662f65d6df9 Mon Sep 17 00:00:00 2001 From: akhandait Date: Tue, 5 Jun 2018 01:16:33 +0530 Subject: [PATCH 60/79] changed names in cmakelists --- src/mlpack/methods/ann/layer/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt index 8d76577727..9ca86a1c74 100644 --- a/src/mlpack/methods/ann/layer/CMakeLists.txt +++ b/src/mlpack/methods/ann/layer/CMakeLists.txt @@ -75,8 +75,8 @@ set(SOURCES recurrent_attention_impl.hpp reinforce_normal.hpp reinforce_normal_impl.hpp - sampling.hpp - sampling_impl.hpp + reparametrization.hpp + reparametrization_impl.hpp select.hpp select_impl.hpp sequential.hpp From 1b93b33ef97591f94fa3a7c39c08e65c8ab6fec9 Mon Sep 17 00:00:00 2001 From: akhandait Date: Tue, 5 Jun 2018 15:53:24 +0530 Subject: [PATCH 61/79] removed build errors --- src/mlpack/methods/ann/layer/layer.hpp | 2 +- src/mlpack/methods/ann/layer/layer_types.hpp | 6 +++--- .../methods/ann/layer/reparametrization.hpp | 17 ++--------------- .../ann/layer/reparametrization_impl.hpp | 7 ++++--- src/mlpack/tests/ann_layer_test.cpp | 8 ++++---- 5 files changed, 14 insertions(+), 26 deletions(-) diff --git a/src/mlpack/methods/ann/layer/layer.hpp b/src/mlpack/methods/ann/layer/layer.hpp index 7eb03f7dfc..78abe96cf4 100644 --- a/src/mlpack/methods/ann/layer/layer.hpp +++ b/src/mlpack/methods/ann/layer/layer.hpp @@ -29,7 +29,7 @@ #include "fast_lstm.hpp" #include "recurrent.hpp" #include "recurrent_attention.hpp" -#include "sampling.hpp" +#include "reparametrization.hpp" #include "sequential.hpp" #include "subview.hpp" #include "concat.hpp" diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index bd67c25084..f8d6ef9e3c 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -36,7 +36,7 @@ #include #include #include -#include +#include #include #include @@ -57,7 +57,7 @@ template class LinearNoBias; template class LSTM; template class GRU; template class FastLSTM; -template class Sampling; +template class Reparametrization; template class VRClassReward; template*, RecurrentAttention*, ReinforceNormal*, - Sampling*, + Reparametrization*, Select*, Sequential*, Subview*, diff --git a/src/mlpack/methods/ann/layer/reparametrization.hpp b/src/mlpack/methods/ann/layer/reparametrization.hpp index 3f79906e09..18862cca7d 100644 --- a/src/mlpack/methods/ann/layer/reparametrization.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization.hpp @@ -94,11 +94,6 @@ class Reparametrization void klBackward(const InputType&& input, OutputType&& output); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -109,15 +104,10 @@ class Reparametrization //! Modify the delta. OutputDataType& Delta() { return delta; } - //! Get the input size. - size_t const& InputSize() const { return inSize; } - //! Modify the input size. - size_t& InputSize() { return inSize; } - //! Get the output size. - size_t const& OutputSize() const { return outSize; } + size_t const& OutputSize() const { return latentSize; } //! Modify the output size. - size_t& OutputSize() { return outSize; } + size_t& OutputSize() { return latentSize; } /** * Serialize the layer @@ -141,9 +131,6 @@ class Reparametrization //! Locally-stored current standard deviation. OutputDataType stdDeviation; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class Reparametrization diff --git a/src/mlpack/methods/ann/layer/reparametrization_impl.hpp b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp index 7e5cb87645..14abae0ef7 100644 --- a/src/mlpack/methods/ann/layer/reparametrization_impl.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp @@ -40,8 +40,9 @@ void Reparametrization::Forward( { if (input.n_rows != 2 * latentSize) { - Log::Fatal << "The output size of layer before the Reparametrization layer should " - << "be 2 * latent size of the Reparametrization layer!" << std::endl; + Log::Fatal << "The output size of layer before the Reparametrization " + << "layer should be 2 * latent size of the Reparametrization layer!" + << std::endl; } mean = input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols - 1); @@ -82,7 +83,7 @@ void Reparametrization::klBackward( OutputType&& output) { output = join_cols(-1 / input.submat(0, 0, latentSize - 1, input.n_cols) - 1, - input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols)) + input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols)); } template diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 04d26600cc..fd331e4ae4 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -1848,12 +1848,12 @@ BOOST_AUTO_TEST_CASE(SubviewIndexTest) } /* - * Simple Sampling module test. + * Simple Reparametrization module test. */ -BOOST_AUTO_TEST_CASE(SimpleSamplingLayerTest) +BOOST_AUTO_TEST_CASE(SimpleReparametrizationLayerTest) { - arma::mat input, output, delta; - Sampling<> module(10, 5); + arma::mat input, output; + Reparametrization<> module(5); // Test the Forward function. input = join_cols(arma::ones(5, 1) * -10, From 11bfec2c758cc1082e0fee099539cd9daddb665d Mon Sep 17 00:00:00 2001 From: akhandait Date: Tue, 5 Jun 2018 19:42:45 +0530 Subject: [PATCH 62/79] corrected kl forward and backward --- src/mlpack/methods/ann/layer/layer_types.hpp | 5 +++++ .../ann/layer/reparametrization_impl.hpp | 19 +++++++++---------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index f8d6ef9e3c..74fe05610f 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -60,6 +60,11 @@ template class FastLSTM; template class Reparametrization; template class VRClassReward; +template +class Reparametrization; + template::Forward( stdDeviation); gaussianSample = arma::randn>(latentSize, input.n_cols); - output = mean + std::move(stdDeviation) % gaussianSample; + output = mean + stdDeviation % gaussianSample; } template template void Reparametrization::Backward( - const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { arma::Mat softplusDer; - SoftplusFunction::Deriv((input - std::move(mean)) / gaussianSample, - softplusDer); + SoftplusFunction::Deriv(std::move(stdDeviation), softplusDer); g = join_cols(gy % std::move(gaussianSample) % std::move(softplusDer), gy); } @@ -71,9 +70,10 @@ double Reparametrization::klForward( const InputType&& input) { stdDeviation = input.submat(0, 0, latentSize - 1, input.n_cols); + mean = input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols); - return -0.5 * arma::accu(arma::log(stdDeviation) - stdDeviation - arma::pow( - input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols), 2) + 1); + return -0.5 * arma::accu(2 * arma::log(stdDeviation) - + arma::pov(stdDeviation, 2) - arma::pow(mean, 2) + 1); } template @@ -82,8 +82,7 @@ void Reparametrization::klBackward( const InputType&& input, OutputType&& output) { - output = join_cols(-1 / input.submat(0, 0, latentSize - 1, input.n_cols) - 1, - input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols)); + output = join_cols(-1 / stdDeviation + stdDeviation, mean); } template From 9d5866744a5f1c09179f88f057413cbb274a6c9b Mon Sep 17 00:00:00 2001 From: akhandait Date: Thu, 7 Jun 2018 18:42:00 +0530 Subject: [PATCH 63/79] added numerical gradient test --- src/mlpack/methods/ann/layer/layer_types.hpp | 1 - .../ann/layer/reparametrization_impl.hpp | 14 +-- src/mlpack/tests/ann_layer_test.cpp | 85 ++++++++++++++----- 3 files changed, 72 insertions(+), 28 deletions(-) diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 74fe05610f..b1f34031a4 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -57,7 +57,6 @@ template class LinearNoBias; template class LSTM; template class GRU; template class FastLSTM; -template class Reparametrization; template class VRClassReward; template::Forward( } mean = input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols - 1); - SoftplusFunction::Fn(input.submat(0, 0, latentSize - 1, input.n_cols - 1), - stdDeviation); + stdDeviation = input.submat(0, 0, latentSize - 1, input.n_cols - 1); + SoftplusFunction::Fn(input.submat(0, 0, latentSize - 1, input.n_cols - 1), + output); gaussianSample = arma::randn>(latentSize, input.n_cols); - output = mean + stdDeviation % gaussianSample; + output = mean + output % gaussianSample; } template @@ -58,10 +59,9 @@ template void Reparametrization::Backward( const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { - arma::Mat softplusDer; - SoftplusFunction::Deriv(std::move(stdDeviation), softplusDer); + SoftplusFunction::Deriv(std::move(stdDeviation), g); - g = join_cols(gy % std::move(gaussianSample) % std::move(softplusDer), gy); + g = join_cols(gy % std::move(gaussianSample) % g, gy); } template @@ -73,7 +73,7 @@ double Reparametrization::klForward( mean = input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols); return -0.5 * arma::accu(2 * arma::log(stdDeviation) - - arma::pov(stdDeviation, 2) - arma::pow(mean, 2) + 1); + arma::pow(stdDeviation, 2) - arma::pow(mean, 2) + 1); } template diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index fd331e4ae4..3503f9cfe7 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -229,7 +229,7 @@ BOOST_AUTO_TEST_CASE(JacobianAddLayerTest) } /** - * Add layer numerically gradient test. + * Add layer numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientAddLayerTest) { @@ -256,7 +256,7 @@ BOOST_AUTO_TEST_CASE(GradientAddLayerTest) double Gradient(arma::mat& gradient) const { - arma::mat output; + // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -535,7 +535,7 @@ BOOST_AUTO_TEST_CASE(JacobianLinearLayerTest) } /** - * Linear layer numerically gradient test. + * Linear layer numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientLinearLayerTest) { @@ -562,7 +562,7 @@ BOOST_AUTO_TEST_CASE(GradientLinearLayerTest) double Gradient(arma::mat& gradient) const { - arma::mat output; + // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -619,7 +619,7 @@ BOOST_AUTO_TEST_CASE(JacobianLinearNoBiasLayerTest) } /** - * LinearNoBias layer numerically gradient test. + * LinearNoBias layer numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientLinearNoBiasLayerTest) { @@ -646,7 +646,7 @@ BOOST_AUTO_TEST_CASE(GradientLinearNoBiasLayerTest) double Gradient(arma::mat& gradient) const { - arma::mat output; + // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -721,7 +721,7 @@ BOOST_AUTO_TEST_CASE(JacobianFlexibleReLULayerTest) } /** - * Flexible ReLU layer numerically gradient test. + * Flexible ReLU layer numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientFlexibleReLULayerTest) { @@ -750,7 +750,7 @@ BOOST_AUTO_TEST_CASE(GradientFlexibleReLULayerTest) double Gradient(arma::mat& gradient) const { - arma::mat output; + // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -957,7 +957,7 @@ BOOST_AUTO_TEST_CASE(GradientLSTMLayerTest) double Gradient(arma::mat& gradient) const { - arma::mat output; + // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -1039,7 +1039,7 @@ BOOST_AUTO_TEST_CASE(GradientFastLSTMLayerTest) double Gradient(arma::mat& gradient) const { - arma::mat output; + // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -1194,7 +1194,7 @@ BOOST_AUTO_TEST_CASE(SimpleConcatLayerTest) } /** - * Concat layer numerically gradient test. + * Concat layer numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientConcatLayerTest) { @@ -1225,7 +1225,7 @@ BOOST_AUTO_TEST_CASE(GradientConcatLayerTest) double Gradient(arma::mat& gradient) const { - arma::mat output; + // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -1394,7 +1394,7 @@ BOOST_AUTO_TEST_CASE(BatchNormTest) } /** - * BatchNorm layer numerically gradient test. + * BatchNorm layer numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientBatchNormTest) { @@ -1423,7 +1423,7 @@ BOOST_AUTO_TEST_CASE(GradientBatchNormTest) double Gradient(arma::mat& gradient) const { - arma::mat output; + // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 256, false); model->Gradient(model->Parameters(), 0, gradient, 256); return error; @@ -1566,7 +1566,7 @@ BOOST_AUTO_TEST_CASE(SimpleTransposedConvolutionLayerTest) } /** - * Transposed Convolution layer numerically gradient test. + * Transposed Convolution layer numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientTransposedConvolutionLayerTest) { @@ -1592,7 +1592,7 @@ BOOST_AUTO_TEST_CASE(GradientTransposedConvolutionLayerTest) double Gradient(arma::mat& gradient) const { - arma::mat output; + // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -1678,7 +1678,7 @@ BOOST_AUTO_TEST_CASE(SimpleAtrousConvolutionLayerTest) } /** - * Atrous Convolution layer numerically gradient test. + * Atrous Convolution layer numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientAtrousConvolutionLayerTest) { @@ -1704,7 +1704,7 @@ BOOST_AUTO_TEST_CASE(GradientAtrousConvolutionLayerTest) double Gradient(arma::mat& gradient) const { - arma::mat output; + // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -1754,7 +1754,7 @@ BOOST_AUTO_TEST_CASE(LayerNormTest) } /** - * LayerNorm layer numerically gradient test. + * LayerNorm layer numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientLayerNormTest) { @@ -1783,7 +1783,7 @@ BOOST_AUTO_TEST_CASE(GradientLayerNormTest) double Gradient(arma::mat& gradient) const { - arma::mat output; + // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 256, false); model->Gradient(model->Parameters(), 0, gradient, 256); return error; @@ -1864,4 +1864,49 @@ BOOST_AUTO_TEST_CASE(SimpleReparametrizationLayerTest) // Test the Backward function. } +/** + * Reparametrization layer numerical gradient test. + */ +BOOST_AUTO_TEST_CASE(GradientReparametrizationLayerTest) +{ + // Linear function gradient instantiation. + struct GradientFunction + { + GradientFunction() + { + input = arma::randu(10, 1); + target = arma::mat("1"); + + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(10, 6); + model->Add >(3); + model->Add >(3, 2); + model->Add >(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + // arma::mat output; + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN, NguyenWidrowInitialization>* model; + arma::mat input, target; + } function; + + BOOST_REQUIRE_LE(CheckGradient(function), 1e-4); +} + BOOST_AUTO_TEST_SUITE_END(); From 309f6428e495ad450943598e219e171090a0fa7f Mon Sep 17 00:00:00 2001 From: akhandait Date: Fri, 8 Jun 2018 16:33:01 +0530 Subject: [PATCH 64/79] gradient check passed, removed redundant lines --- .../methods/ann/layer/reparametrization.hpp | 9 ++++-- .../ann/layer/reparametrization_impl.hpp | 29 +++++++++++-------- src/mlpack/tests/ann_layer_test.cpp | 14 +-------- 3 files changed, 24 insertions(+), 28 deletions(-) diff --git a/src/mlpack/methods/ann/layer/reparametrization.hpp b/src/mlpack/methods/ann/layer/reparametrization.hpp index 18862cca7d..1a91341fa2 100644 --- a/src/mlpack/methods/ann/layer/reparametrization.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization.hpp @@ -45,7 +45,7 @@ class Reparametrization * * @param layerSize The number of output units. */ - Reparametrization(const size_t latentSize); + Reparametrization(const size_t latentSize, const bool stochastic = true); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -119,6 +119,9 @@ class Reparametrization //! Locally-stored number of output units. size_t latentSize; + //! If false, sample will be constant. + bool stochastic; + //! Locally-stored delta object. OutputDataType delta; @@ -128,8 +131,8 @@ class Reparametrization //! Locally-stored current mean. OutputDataType mean; - //! Locally-stored current standard deviation. - OutputDataType stdDeviation; + //! Locally-stored pre Standard Deviation, after softplus gives Standard Deviation. + OutputDataType preStdDev; //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/reparametrization_impl.hpp b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp index 117ae264eb..8244dc7693 100644 --- a/src/mlpack/methods/ann/layer/reparametrization_impl.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp @@ -27,8 +27,10 @@ Reparametrization::Reparametrization() template Reparametrization::Reparametrization( - const size_t latentSize) : - latentSize(latentSize) + const size_t latentSize, + const bool stochastic) : + latentSize(latentSize), + stochastic(stochastic) { // Nothing to do here. } @@ -46,11 +48,14 @@ void Reparametrization::Forward( } mean = input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols - 1); - stdDeviation = input.submat(0, 0, latentSize - 1, input.n_cols - 1); + preStdDev = input.submat(0, 0, latentSize - 1, input.n_cols - 1); - SoftplusFunction::Fn(input.submat(0, 0, latentSize - 1, input.n_cols - 1), - output); - gaussianSample = arma::randn>(latentSize, input.n_cols); + if (stochastic) + gaussianSample = arma::randn>(latentSize, input.n_cols); + else + gaussianSample = arma::ones>(latentSize, input.n_cols) * 0.7; + + SoftplusFunction::Fn(preStdDev, output); output = mean + output % gaussianSample; } @@ -59,8 +64,7 @@ template void Reparametrization::Backward( const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { - SoftplusFunction::Deriv(std::move(stdDeviation), g); - + SoftplusFunction::Deriv(std::move(preStdDev), g); g = join_cols(gy % std::move(gaussianSample) % g, gy); } @@ -69,11 +73,11 @@ template double Reparametrization::klForward( const InputType&& input) { - stdDeviation = input.submat(0, 0, latentSize - 1, input.n_cols); + preStdDev = input.submat(0, 0, latentSize - 1, input.n_cols); mean = input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols); - return -0.5 * arma::accu(2 * arma::log(stdDeviation) - - arma::pow(stdDeviation, 2) - arma::pow(mean, 2) + 1); + return -0.5 * arma::accu(2 * arma::log(preStdDev) - + arma::pow(preStdDev, 2) - arma::pow(mean, 2) + 1); } template @@ -82,7 +86,7 @@ void Reparametrization::klBackward( const InputType&& input, OutputType&& output) { - output = join_cols(-1 / stdDeviation + stdDeviation, mean); + output = join_cols(-1 / preStdDev + preStdDev, mean); } template @@ -91,6 +95,7 @@ void Reparametrization::serialize( Archive& ar, const unsigned int /* version */) { ar & BOOST_SERIALIZATION_NVP(latentSize); + ar & BOOST_SERIALIZATION_NVP(stochastic); } } // namespace ann diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 3503f9cfe7..6c98d09434 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -256,7 +256,6 @@ BOOST_AUTO_TEST_CASE(GradientAddLayerTest) double Gradient(arma::mat& gradient) const { - // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -562,7 +561,6 @@ BOOST_AUTO_TEST_CASE(GradientLinearLayerTest) double Gradient(arma::mat& gradient) const { - // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -646,7 +644,6 @@ BOOST_AUTO_TEST_CASE(GradientLinearNoBiasLayerTest) double Gradient(arma::mat& gradient) const { - // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -750,7 +747,6 @@ BOOST_AUTO_TEST_CASE(GradientFlexibleReLULayerTest) double Gradient(arma::mat& gradient) const { - // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -957,7 +953,6 @@ BOOST_AUTO_TEST_CASE(GradientLSTMLayerTest) double Gradient(arma::mat& gradient) const { - // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -1039,7 +1034,6 @@ BOOST_AUTO_TEST_CASE(GradientFastLSTMLayerTest) double Gradient(arma::mat& gradient) const { - // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -1225,7 +1219,6 @@ BOOST_AUTO_TEST_CASE(GradientConcatLayerTest) double Gradient(arma::mat& gradient) const { - // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -1423,7 +1416,6 @@ BOOST_AUTO_TEST_CASE(GradientBatchNormTest) double Gradient(arma::mat& gradient) const { - // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 256, false); model->Gradient(model->Parameters(), 0, gradient, 256); return error; @@ -1592,7 +1584,6 @@ BOOST_AUTO_TEST_CASE(GradientTransposedConvolutionLayerTest) double Gradient(arma::mat& gradient) const { - // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -1704,7 +1695,6 @@ BOOST_AUTO_TEST_CASE(GradientAtrousConvolutionLayerTest) double Gradient(arma::mat& gradient) const { - // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -1783,7 +1773,6 @@ BOOST_AUTO_TEST_CASE(GradientLayerNormTest) double Gradient(arma::mat& gradient) const { - // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 256, false); model->Gradient(model->Parameters(), 0, gradient, 256); return error; @@ -1882,7 +1871,7 @@ BOOST_AUTO_TEST_CASE(GradientReparametrizationLayerTest) model->Responses() = target; model->Add >(); model->Add >(10, 6); - model->Add >(3); + model->Add >(3, false); model->Add >(3, 2); model->Add >(); } @@ -1894,7 +1883,6 @@ BOOST_AUTO_TEST_CASE(GradientReparametrizationLayerTest) double Gradient(arma::mat& gradient) const { - // arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; From 88ae069b51a0b51686275e961b0bd0628d9ca551 Mon Sep 17 00:00:00 2001 From: akhandait Date: Sat, 9 Jun 2018 01:53:08 +0530 Subject: [PATCH 65/79] corrections made in kl, more tests added --- .../methods/ann/layer/reparametrization.hpp | 11 +++-- .../ann/layer/reparametrization_impl.hpp | 16 +++---- src/mlpack/tests/ann_layer_test.cpp | 45 +++++++++++++++++-- 3 files changed, 57 insertions(+), 15 deletions(-) diff --git a/src/mlpack/methods/ann/layer/reparametrization.hpp b/src/mlpack/methods/ann/layer/reparametrization.hpp index 1a91341fa2..6eb23e1f12 100644 --- a/src/mlpack/methods/ann/layer/reparametrization.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization.hpp @@ -90,9 +90,8 @@ class Reparametrization * @param gy The backpropagated error. * @param g The calculated gradient. */ - template - void klBackward(const InputType&& input, - OutputType&& output); + template + void klBackward(OutputType&& output); //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } @@ -131,9 +130,13 @@ class Reparametrization //! Locally-stored current mean. OutputDataType mean; - //! Locally-stored pre Standard Deviation, after softplus gives Standard Deviation. + //! Locally-stored pre standard deviation. + //! After softplus activation gives standard deviation. OutputDataType preStdDev; + //! Locally-stored current standard deviation. + OutputDataType stdDev; + //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class Reparametrization diff --git a/src/mlpack/methods/ann/layer/reparametrization_impl.hpp b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp index 8244dc7693..9e413fcd80 100644 --- a/src/mlpack/methods/ann/layer/reparametrization_impl.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp @@ -56,7 +56,8 @@ void Reparametrization::Forward( gaussianSample = arma::ones>(latentSize, input.n_cols) * 0.7; SoftplusFunction::Fn(preStdDev, output); - output = mean + output % gaussianSample; + output %= gaussianSample; + output += mean; } template @@ -73,20 +74,19 @@ template double Reparametrization::klForward( const InputType&& input) { - preStdDev = input.submat(0, 0, latentSize - 1, input.n_cols); - mean = input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols); + stdDev = input.submat(0, 0, latentSize - 1, input.n_cols - 1); + mean = input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols - 1); - return -0.5 * arma::accu(2 * arma::log(preStdDev) - - arma::pow(preStdDev, 2) - arma::pow(mean, 2) + 1); + return -0.5 * arma::accu(2 * arma::log(stdDev) - + arma::pow(stdDev, 2) - arma::pow(mean, 2) + 1); } template -template +template void Reparametrization::klBackward( - const InputType&& input, OutputType&& output) { - output = join_cols(-1 / preStdDev + preStdDev, mean); + output = join_cols(-1 / stdDev + stdDev, mean); } template diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 6c98d09434..0aff073324 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -1841,16 +1841,38 @@ BOOST_AUTO_TEST_CASE(SubviewIndexTest) */ BOOST_AUTO_TEST_CASE(SimpleReparametrizationLayerTest) { - arma::mat input, output; + arma::mat input, output, delta; Reparametrization<> module(5); // Test the Forward function. - input = join_cols(arma::ones(5, 1) * -10, + input = join_cols(arma::ones(5, 1) * -20, arma::zeros(5, 1)); module.Forward(std::move(input), std::move(output)); - BOOST_REQUIRE_LE(arma::accu(output), 1e-3); + BOOST_REQUIRE_LE(arma::accu(output), 1e-5); // Test the Backward function. + arma::mat gy = arma::zeros(5, 1); + module.Backward(std::move(input), std::move(gy), std::move(delta)); + BOOST_REQUIRE_EQUAL(arma::accu(delta), 0); +} + +/** + * Jacobian Reparametrization module test. + */ +BOOST_AUTO_TEST_CASE(JacobianReparametrizationLayerTest) +{ + for (size_t i = 0; i < 5; i++) + { + const size_t inputElementsHalf = math::RandInt(2, 1000); + + arma::mat input; + input.set_size(inputElementsHalf * 2, 1); + + Reparametrization<> module(inputElementsHalf, false); + + double error = JacobianTest(module, input); + BOOST_REQUIRE_LE(error, 1e-5); + } } /** @@ -1897,4 +1919,21 @@ BOOST_AUTO_TEST_CASE(GradientReparametrizationLayerTest) BOOST_REQUIRE_LE(CheckGradient(function), 1e-4); } +/** + * Simple Reparametrization module KL divergence test. + */ +BOOST_AUTO_TEST_CASE(SimpleReparametrizationLayerKlTest) +{ + arma::mat input, output; + Reparametrization<> module(5); + + // Test the Forward function. + input = join_cols(arma::ones(5, 1), arma::zeros(5, 1)); + BOOST_REQUIRE_EQUAL(module.klForward(std::move(input)), 0); + + // Test the Backward function. + module.klBackward(output); + BOOST_REQUIRE_EQUAL(arma::accu(std::move(output)), 0); +} + BOOST_AUTO_TEST_SUITE_END(); From dcfef4f750f955219431adfc6b0d54e5c7219f0f Mon Sep 17 00:00:00 2001 From: akhandait Date: Sat, 9 Jun 2018 18:29:47 +0530 Subject: [PATCH 66/79] loss visitor added --- src/mlpack/methods/ann/layer/layer_traits.hpp | 4 ++++ src/mlpack/methods/ann/layer/reparametrization.hpp | 6 ++++++ src/mlpack/methods/ann/visitor/CMakeLists.txt | 2 ++ src/mlpack/methods/ann/visitor/output_height_visitor.hpp | 4 ++-- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/layer/layer_traits.hpp b/src/mlpack/methods/ann/layer/layer_traits.hpp index 02691f7b7a..f69e9d3c94 100644 --- a/src/mlpack/methods/ann/layer/layer_traits.hpp +++ b/src/mlpack/methods/ann/layer/layer_traits.hpp @@ -104,6 +104,10 @@ HAS_MEM_FUNC(InputHeight, HasInputHeight); // can use with SFINAE to catch when a type has a Rho() function. HAS_MEM_FUNC(Rho, HasRho); +// This gives us a HasLoss type (where U is a function pointer) we +// can use with SFINAE to catch when a type has a Loss() function. +HAS_MEM_FUNC(Loss, HasLoss); + } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/reparametrization.hpp b/src/mlpack/methods/ann/layer/reparametrization.hpp index 6eb23e1f12..af514c8d79 100644 --- a/src/mlpack/methods/ann/layer/reparametrization.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization.hpp @@ -108,6 +108,12 @@ class Reparametrization //! Modify the output size. size_t& OutputSize() { return latentSize; } + //! Get the KL divergence with standard normal. + double const& Loss() const + { + return klForward(join_cols(stdDev, mean)); + } + /** * Serialize the layer */ diff --git a/src/mlpack/methods/ann/visitor/CMakeLists.txt b/src/mlpack/methods/ann/visitor/CMakeLists.txt index 187a2bff1b..d11ddf640d 100644 --- a/src/mlpack/methods/ann/visitor/CMakeLists.txt +++ b/src/mlpack/methods/ann/visitor/CMakeLists.txt @@ -25,6 +25,8 @@ set(SOURCES gradient_zero_visitor_impl.hpp load_output_parameter_visitor.hpp load_output_parameter_visitor_impl.hpp + loss_visitor.hpp + loss_visitor_impl.hpp output_height_visitor.hpp output_height_visitor_impl.hpp output_parameter_visitor.hpp diff --git a/src/mlpack/methods/ann/visitor/output_height_visitor.hpp b/src/mlpack/methods/ann/visitor/output_height_visitor.hpp index b2d968d649..05f9c67e2a 100644 --- a/src/mlpack/methods/ann/visitor/output_height_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/output_height_visitor.hpp @@ -22,7 +22,7 @@ namespace mlpack { namespace ann { /** - * OutputWidthVisitor exposes the OutputHeight() method of the given module. + * OutputHeightVisitor exposes the OutputHeight() method of the given module. */ class OutputHeightVisitor : public boost::static_visitor { @@ -55,7 +55,7 @@ class OutputHeightVisitor : public boost::static_visitor HasModelCheck::value, size_t>::type LayerOutputHeight(T* layer) const; - //! Return the output height if the module implement the Model() or + //! Return the output height if the module implements the Model() or //! InputHeight() function. template typename std::enable_if< From 65c8015505f4446a303309c5a60ef78dd12423c0 Mon Sep 17 00:00:00 2001 From: akhandait Date: Sat, 9 Jun 2018 18:30:06 +0530 Subject: [PATCH 67/79] loss visitor added --- .../methods/ann/visitor/loss_visitor.hpp | 69 ++++++++++++++ .../methods/ann/visitor/loss_visitor_impl.hpp | 94 +++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 src/mlpack/methods/ann/visitor/loss_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/loss_visitor_impl.hpp diff --git a/src/mlpack/methods/ann/visitor/loss_visitor.hpp b/src/mlpack/methods/ann/visitor/loss_visitor.hpp new file mode 100644 index 0000000000..88d77ef3a3 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/loss_visitor.hpp @@ -0,0 +1,69 @@ +/** + * @file loss_visitor.hpp + * @author Atharva Khandait + * + * This file provides an abstraction for the Loss() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_LOSS_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_LOSS_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * LossVisitor exposes the Loss() method of the given module. + */ +class LossVisitor : public boost::static_visitor +{ + public: + //! Return the Loss. + template + double operator()(LayerType* layer) const; + + private: + //! Return 0 if the module doesn't implement the Loss() or Model() function. + template + typename std::enable_if< + !HasLoss::value && + !HasModelCheck::value, double>::type + LayerLoss(T* layer) const; + + //! Return the output height if the module implements the Loss() function. + template + typename std::enable_if< + HasLoss::value && + !HasModelCheck::value, double>::type + LayerLoss(T* layer) const; + + //! Return the loss if the module implements the Model() function. + template + typename std::enable_if< + !HasLoss::value && + HasModelCheck::value, double>::type + LayerLoss(T* layer) const; + + //! Return the loss if the module implements the Model() or loss() function. + template + typename std::enable_if< + HasLoss::value && + HasModelCheck::value, double>::type + LayerLoss(T* layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "kl_divergence_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/loss_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/loss_visitor_impl.hpp new file mode 100644 index 0000000000..a363457360 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/loss_visitor_impl.hpp @@ -0,0 +1,94 @@ +/** + * @file loss_visitor_impl.hpp + * @author Atharva Khandait + * + * Implementation of the Loss() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_LOSS_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_LOSS_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "loss_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! LossVisitor visitor class. +template +inline double LossVisitor::operator()(LayerType* layer) const +{ + return LayerLoss(layer); +} + +template +inline typename std::enable_if< + !HasLoss::value && + !HasModelCheck::value, double>::type +LossVisitor::LayerLoss(T* /* layer */) const +{ + return 0; +} + +template +inline typename std::enable_if< + HasLoss::value && + !HasModelCheck::value, double>::type +LossVisitor::LayerLoss(T* layer) const +{ + return layer->Loss(); +} + +template +inline typename std::enable_if< + !HasLoss::value && + HasModelCheck::value, double>::type +LossVisitor::LayerLoss(T* layer) const +{ + for (size_t i = 0; i < layer->Model().size(); ++i) + { + double Loss = boost::apply_visitor(LossVisitor(), + layer->Model()[layer->Model().size() - 1 - i]); + + if (Loss != 0) + { + return Loss; + } + } + + return 0; +} + +template +inline typename std::enable_if< + HasLoss::value && + HasModelCheck::value, double>::type +LossVisitor::LayerLoss(T* layer) const +{ + double Loss = layer->Loss(); + + if (Loss == 0) + { + for (size_t i = 0; i < layer->Model().size(); ++i) + { + Loss = boost::apply_visitor(LossVisitor(), + layer->Model()[layer->Model().size() - 1 - i]); + + if (Loss != 0) + { + return Loss; + } + } + } + + return Loss; +} + +} // namespace ann +} // namespace mlpack + +#endif From 312b155f188147d5542e09fdb864016aad6a7a9b Mon Sep 17 00:00:00 2001 From: akhandait Date: Mon, 11 Jun 2018 23:05:16 +0530 Subject: [PATCH 68/79] extra loss added to evaluate, backward --- src/mlpack/methods/ann/ffn.hpp | 4 ++++ src/mlpack/methods/ann/ffn_impl.hpp | 10 ++++++++++ src/mlpack/methods/ann/layer/reparametrization.hpp | 5 +++-- .../methods/ann/layer/reparametrization_impl.hpp | 14 ++++++++------ src/mlpack/methods/ann/visitor/loss_visitor.hpp | 10 +++++----- .../methods/ann/visitor/loss_visitor_impl.hpp | 8 ++++---- src/mlpack/tests/ann_layer_test.cpp | 2 +- 7 files changed, 35 insertions(+), 18 deletions(-) diff --git a/src/mlpack/methods/ann/ffn.hpp b/src/mlpack/methods/ann/ffn.hpp index 46db64b77a..840a1c453b 100644 --- a/src/mlpack/methods/ann/ffn.hpp +++ b/src/mlpack/methods/ann/ffn.hpp @@ -23,6 +23,7 @@ #include "visitor/reset_visitor.hpp" #include "visitor/weight_size_visitor.hpp" #include "visitor/copy_visitor.hpp" +#include "visitor/loss_visitor.hpp" #include "init_rules/network_init.hpp" @@ -371,6 +372,9 @@ class FFN //! Locally-stored output height visitor. OutputHeightVisitor outputHeightVisitor; + //! Locally-stored loss visitor + LossVisitor lossVisitor; + //! Locally-stored reset visitor. ResetVisitor resetVisitor; diff --git a/src/mlpack/methods/ann/ffn_impl.hpp b/src/mlpack/methods/ann/ffn_impl.hpp index c0811f4716..0a00f05466 100644 --- a/src/mlpack/methods/ann/ffn_impl.hpp +++ b/src/mlpack/methods/ann/ffn_impl.hpp @@ -132,6 +132,11 @@ double FFN::Backward( double res = outputLayer.Forward(std::move(boost::apply_visitor( outputParameterVisitor, network.back())), std::move(targets)); + for (size_t i = 0; i < network.size(); ++i) + { + res += boost::apply_visitor(lossVisitor, network[i]); + } + outputLayer.Backward(std::move(boost::apply_visitor(outputParameterVisitor, network.back())), std::move(targets), std::move(error)); @@ -212,6 +217,11 @@ double FFN::Evaluate( std::move(boost::apply_visitor(outputParameterVisitor, network.back())), std::move(responses.cols(begin, begin + batchSize - 1))); + for (size_t i = 0; i < network.size(); ++i) + { + res += boost::apply_visitor(lossVisitor, network[i]); + } + return res; } diff --git a/src/mlpack/methods/ann/layer/reparametrization.hpp b/src/mlpack/methods/ann/layer/reparametrization.hpp index af514c8d79..73a47115b1 100644 --- a/src/mlpack/methods/ann/layer/reparametrization.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization.hpp @@ -109,9 +109,10 @@ class Reparametrization size_t& OutputSize() { return latentSize; } //! Get the KL divergence with standard normal. - double const& Loss() const + double Loss() { - return klForward(join_cols(stdDev, mean)); + OutputDataType input = join_cols(stdDev, mean); + return klForward(std::move(input)); } /** diff --git a/src/mlpack/methods/ann/layer/reparametrization_impl.hpp b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp index 9e413fcd80..7e0a62345d 100644 --- a/src/mlpack/methods/ann/layer/reparametrization_impl.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp @@ -55,9 +55,8 @@ void Reparametrization::Forward( else gaussianSample = arma::ones>(latentSize, input.n_cols) * 0.7; - SoftplusFunction::Fn(preStdDev, output); - output %= gaussianSample; - output += mean; + SoftplusFunction::Fn(preStdDev, stdDev); + output = mean + stdDev % gaussianSample; } template @@ -65,8 +64,10 @@ template void Reparametrization::Backward( const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { - SoftplusFunction::Deriv(std::move(preStdDev), g); - g = join_cols(gy % std::move(gaussianSample) % g, gy); + SoftplusFunction::Deriv(preStdDev, g); + arma::Mat klBack; + klBackward(std::move(klBack)); + g = join_cols(gy % std::move(gaussianSample) % g, gy) + std::move(klBack); } template @@ -86,7 +87,8 @@ template void Reparametrization::klBackward( OutputType&& output) { - output = join_cols(-1 / stdDev + stdDev, mean); + SoftplusFunction::Deriv(preStdDev, output); + output = join_cols((-1 / stdDev + stdDev) % output, mean); } template diff --git a/src/mlpack/methods/ann/visitor/loss_visitor.hpp b/src/mlpack/methods/ann/visitor/loss_visitor.hpp index 88d77ef3a3..31b05e6c56 100644 --- a/src/mlpack/methods/ann/visitor/loss_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/loss_visitor.hpp @@ -34,28 +34,28 @@ class LossVisitor : public boost::static_visitor //! Return 0 if the module doesn't implement the Loss() or Model() function. template typename std::enable_if< - !HasLoss::value && + !HasLoss::value && !HasModelCheck::value, double>::type LayerLoss(T* layer) const; //! Return the output height if the module implements the Loss() function. template typename std::enable_if< - HasLoss::value && + HasLoss::value && !HasModelCheck::value, double>::type LayerLoss(T* layer) const; //! Return the loss if the module implements the Model() function. template typename std::enable_if< - !HasLoss::value && + !HasLoss::value && HasModelCheck::value, double>::type LayerLoss(T* layer) const; //! Return the loss if the module implements the Model() or loss() function. template typename std::enable_if< - HasLoss::value && + HasLoss::value && HasModelCheck::value, double>::type LayerLoss(T* layer) const; }; @@ -64,6 +64,6 @@ class LossVisitor : public boost::static_visitor } // namespace mlpack // Include implementation. -#include "kl_divergence_visitor_impl.hpp" +#include "loss_visitor_impl.hpp" #endif diff --git a/src/mlpack/methods/ann/visitor/loss_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/loss_visitor_impl.hpp index a363457360..fa19f63948 100644 --- a/src/mlpack/methods/ann/visitor/loss_visitor_impl.hpp +++ b/src/mlpack/methods/ann/visitor/loss_visitor_impl.hpp @@ -27,7 +27,7 @@ inline double LossVisitor::operator()(LayerType* layer) const template inline typename std::enable_if< - !HasLoss::value && + !HasLoss::value && !HasModelCheck::value, double>::type LossVisitor::LayerLoss(T* /* layer */) const { @@ -36,7 +36,7 @@ LossVisitor::LayerLoss(T* /* layer */) const template inline typename std::enable_if< - HasLoss::value && + HasLoss::value && !HasModelCheck::value, double>::type LossVisitor::LayerLoss(T* layer) const { @@ -45,7 +45,7 @@ LossVisitor::LayerLoss(T* layer) const template inline typename std::enable_if< - !HasLoss::value && + !HasLoss::value && HasModelCheck::value, double>::type LossVisitor::LayerLoss(T* layer) const { @@ -65,7 +65,7 @@ LossVisitor::LayerLoss(T* layer) const template inline typename std::enable_if< - HasLoss::value && + HasLoss::value && HasModelCheck::value, double>::type LossVisitor::LayerLoss(T* layer) const { diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 0aff073324..c36007f711 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -1853,7 +1853,7 @@ BOOST_AUTO_TEST_CASE(SimpleReparametrizationLayerTest) // Test the Backward function. arma::mat gy = arma::zeros(5, 1); module.Backward(std::move(input), std::move(gy), std::move(delta)); - BOOST_REQUIRE_EQUAL(arma::accu(delta), 0); + BOOST_REQUIRE(arma::accu(delta) != 0); // klBackward will be added. } /** From 3dbbe138012d336277a04b58dd67943bcdfef6b9 Mon Sep 17 00:00:00 2001 From: akhandait Date: Tue, 12 Jun 2018 02:13:09 +0530 Subject: [PATCH 69/79] removed simple kl test --- src/mlpack/tests/ann_layer_test.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index c36007f711..13d5fc4f98 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -1919,21 +1919,21 @@ BOOST_AUTO_TEST_CASE(GradientReparametrizationLayerTest) BOOST_REQUIRE_LE(CheckGradient(function), 1e-4); } -/** - * Simple Reparametrization module KL divergence test. - */ -BOOST_AUTO_TEST_CASE(SimpleReparametrizationLayerKlTest) -{ - arma::mat input, output; - Reparametrization<> module(5); +// /** +// * Simple Reparametrization module KL divergence test. +// */ +// BOOST_AUTO_TEST_CASE(SimpleReparametrizationLayerKlTest) +// { +// arma::mat input, output; +// Reparametrization<> module(5); - // Test the Forward function. - input = join_cols(arma::ones(5, 1), arma::zeros(5, 1)); - BOOST_REQUIRE_EQUAL(module.klForward(std::move(input)), 0); +// // Test the Forward function. +// input = join_cols(arma::ones(5, 1), arma::zeros(5, 1)); +// BOOST_REQUIRE_EQUAL(module.klForward(std::move(input)), 0); - // Test the Backward function. - module.klBackward(output); - BOOST_REQUIRE_EQUAL(arma::accu(std::move(output)), 0); -} +// // Test the Backward function. +// module.klBackward(output); +// BOOST_REQUIRE(arma::accu(std::move(output)) != 0); +// } BOOST_AUTO_TEST_SUITE_END(); From 059fe89c56208334609f8ee342972e3f1a46a375 Mon Sep 17 00:00:00 2001 From: akhandait Date: Tue, 12 Jun 2018 02:14:04 +0530 Subject: [PATCH 70/79] removed simple kl test --- src/mlpack/tests/ann_layer_test.cpp | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 13d5fc4f98..4e7ca8567a 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -1919,21 +1919,4 @@ BOOST_AUTO_TEST_CASE(GradientReparametrizationLayerTest) BOOST_REQUIRE_LE(CheckGradient(function), 1e-4); } -// /** -// * Simple Reparametrization module KL divergence test. -// */ -// BOOST_AUTO_TEST_CASE(SimpleReparametrizationLayerKlTest) -// { -// arma::mat input, output; -// Reparametrization<> module(5); - -// // Test the Forward function. -// input = join_cols(arma::ones(5, 1), arma::zeros(5, 1)); -// BOOST_REQUIRE_EQUAL(module.klForward(std::move(input)), 0); - -// // Test the Backward function. -// module.klBackward(output); -// BOOST_REQUIRE(arma::accu(std::move(output)) != 0); -// } - BOOST_AUTO_TEST_SUITE_END(); From e357e8f0f03f1af17d868a05c898b66d160f80c3 Mon Sep 17 00:00:00 2001 From: akhandait Date: Wed, 13 Jun 2018 21:02:14 +0530 Subject: [PATCH 71/79] includeKl boolean added, tests added --- .../methods/ann/layer/reparametrization.hpp | 7 +++- .../ann/layer/reparametrization_impl.hpp | 18 ++++++--- src/mlpack/tests/ann_layer_test.cpp | 37 ++++++++++++++++++- 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/src/mlpack/methods/ann/layer/reparametrization.hpp b/src/mlpack/methods/ann/layer/reparametrization.hpp index 73a47115b1..73ec806e96 100644 --- a/src/mlpack/methods/ann/layer/reparametrization.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization.hpp @@ -45,7 +45,9 @@ class Reparametrization * * @param layerSize The number of output units. */ - Reparametrization(const size_t latentSize, const bool stochastic = true); + Reparametrization(const size_t latentSize, + const bool stochastic = true, + const bool includeKl = true); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -128,6 +130,9 @@ class Reparametrization //! If false, sample will be constant. bool stochastic; + //! If false, KL error will not be included in Backward function. + bool includeKl; + //! Locally-stored delta object. OutputDataType delta; diff --git a/src/mlpack/methods/ann/layer/reparametrization_impl.hpp b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp index 7e0a62345d..58264218e6 100644 --- a/src/mlpack/methods/ann/layer/reparametrization_impl.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp @@ -28,9 +28,11 @@ Reparametrization::Reparametrization() template Reparametrization::Reparametrization( const size_t latentSize, - const bool stochastic) : + const bool stochastic, + const bool includeKl) : latentSize(latentSize), - stochastic(stochastic) + stochastic(stochastic), + includeKl(includeKl) { // Nothing to do here. } @@ -65,9 +67,15 @@ void Reparametrization::Backward( const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { SoftplusFunction::Deriv(preStdDev, g); - arma::Mat klBack; - klBackward(std::move(klBack)); - g = join_cols(gy % std::move(gaussianSample) % g, gy) + std::move(klBack); + + if (includeKl) + { + arma::Mat klBack; + klBackward(std::move(klBack)); + g = join_cols(gy % std::move(gaussianSample) % g, gy) + std::move(klBack); + } + else + g = join_cols(gy % std::move(gaussianSample) % g, gy); } template diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 4e7ca8567a..b0c4e100c3 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -1856,6 +1856,41 @@ BOOST_AUTO_TEST_CASE(SimpleReparametrizationLayerTest) BOOST_REQUIRE(arma::accu(delta) != 0); // klBackward will be added. } +/** + * Reparametrization module stochastic boolean test. + */ +BOOST_AUTO_TEST_CASE(ReparametrizationLayerStochasticTest) +{ + arma::mat input, outputA, outputB; + Reparametrization<> module(5, false); + + input = join_cols(arma::ones(5, 1), + arma::zeros(5, 1)); + + // Test if two forward passes generate same output. + module.Forward(std::move(input), std::move(outputA)); + module.Forward(std::move(input), std::move(outputB)); + + CheckMatrices(std::move(outputA), std::move(outputB)); +} + +/** + * Reparametrization module includeKl boolean test. + */ +BOOST_AUTO_TEST_CASE(ReparametrizationLayerIncludeKlTest) +{ + arma::mat input, output, gy, delta; + Reparametrization<> module(5, true, false); + + input = join_cols(arma::ones(5, 1), + arma::zeros(5, 1)); + module.Forward(std::move(input), std::move(output)); + gy = arma::zeros(output.n_rows, output.n_cols); + module.Backward(std::move(output), std::move(gy), std::move(delta)); + + BOOST_REQUIRE_EQUAL(arma::accu(std::move(delta)), 0); +} + /** * Jacobian Reparametrization module test. */ @@ -1868,7 +1903,7 @@ BOOST_AUTO_TEST_CASE(JacobianReparametrizationLayerTest) arma::mat input; input.set_size(inputElementsHalf * 2, 1); - Reparametrization<> module(inputElementsHalf, false); + Reparametrization<> module(inputElementsHalf, false, false); double error = JacobianTest(module, input); BOOST_REQUIRE_LE(error, 1e-5); From 342972cfb33fc66ec7b665e86e2e73cb54013818 Mon Sep 17 00:00:00 2001 From: Shikhar Jaiswal Date: Wed, 13 Jun 2018 02:14:47 +0530 Subject: [PATCH 72/79] Implement Batch Support --- src/mlpack/core/math/make_alias.hpp | 13 ++ src/mlpack/methods/ann/gan_impl.hpp | 48 ++++--- .../methods/ann/layer/atrous_convolution.hpp | 7 +- .../ann/layer/atrous_convolution_impl.hpp | 132 ++++++++++-------- .../ann/layer/bilinear_interpolation.hpp | 5 +- .../ann/layer/bilinear_interpolation_impl.hpp | 36 +++-- src/mlpack/methods/ann/layer/convolution.hpp | 7 +- .../methods/ann/layer/convolution_impl.hpp | 104 ++++++++------ src/mlpack/methods/ann/layer/max_pooling.hpp | 7 +- .../methods/ann/layer/max_pooling_impl.hpp | 21 ++- src/mlpack/methods/ann/layer/mean_pooling.hpp | 7 +- .../methods/ann/layer/mean_pooling_impl.hpp | 23 +-- .../ann/layer/transposed_convolution.hpp | 7 +- .../ann/layer/transposed_convolution_impl.hpp | 82 +++++++---- src/mlpack/tests/gan_test.cpp | 6 +- 15 files changed, 310 insertions(+), 195 deletions(-) diff --git a/src/mlpack/core/math/make_alias.hpp b/src/mlpack/core/math/make_alias.hpp index 7ed0687141..a83620226f 100644 --- a/src/mlpack/core/math/make_alias.hpp +++ b/src/mlpack/core/math/make_alias.hpp @@ -16,6 +16,19 @@ namespace mlpack { namespace math { +/** + * Make an alias of a dense cube. If strict is true, then the alias cannot be + * resized or pointed at new memory. + */ +template +arma::Cube MakeAlias(arma::Cube& input, + const bool strict = true) +{ + // Use the advanced constructor. + return arma::Cube(input.memptr(), input.n_rows, input.n_cols, + input.n_slices, false, strict); +} + /** * Make an alias of a dense matrix. If strict is true, then the alias cannot be * resized or pointed at new memory. diff --git a/src/mlpack/methods/ann/gan_impl.hpp b/src/mlpack/methods/ann/gan_impl.hpp index 673cb74b68..1160dba6f9 100644 --- a/src/mlpack/methods/ann/gan_impl.hpp +++ b/src/mlpack/methods/ann/gan_impl.hpp @@ -60,19 +60,21 @@ GAN::GAN( responses.set_size(1, predictors.n_cols); responses.ones(); - discriminator.predictors.set_size(predictors.n_rows, predictors.n_cols + 1); + discriminator.predictors.set_size(predictors.n_rows, + predictors.n_cols + batchSize); discriminator.predictors.cols(0, predictors.n_cols - 1) = predictors; - discriminator.responses.set_size(1, predictors.n_cols + 1); + discriminator.responses.set_size(1, predictors.n_cols + batchSize); discriminator.responses.ones(); - discriminator.responses(predictors.n_cols) = 0; + discriminator.responses.cols(predictors.n_cols, + predictors.n_cols + batchSize - 1) = arma::zeros(1, batchSize); numFunctions = predictors.n_cols; - noise.set_size(noiseDim, 1); + noise.set_size(noiseDim, batchSize); - generator.predictors.set_size(noiseDim, 1); - generator.responses.set_size(predictors.n_rows, 1); + generator.predictors.set_size(noiseDim, batchSize); + generator.responses.set_size(predictors.n_rows, batchSize); } template @@ -98,7 +100,7 @@ void GAN::Reset() generator.Parameters() = arma::mat(parameter.memptr(), genWeights, 1, false, false); discriminator.Parameters() = arma::mat(parameter.memptr() + genWeights, - discWeights, 1 , false, false); + discWeights, 1, false, false); // Initialize the parameters generator networkInit.Initialize(generator.network, parameter); @@ -127,8 +129,10 @@ double GAN::Evaluate( if (!reset) Reset(); - currentInput = this->predictors.unsafe_col(i); - currentTarget = this->responses.unsafe_col(i); + currentInput = arma::mat(predictors.memptr() + (i * predictors.n_rows), + predictors.n_rows, batchSize, false, false); + currentTarget = arma::mat(responses.memptr() + i, 1, batchSize, false, + false); discriminator.Forward(std::move(currentInput)); double res = discriminator.outputLayer.Forward( @@ -139,12 +143,15 @@ double GAN::Evaluate( noise.imbue( [&]() { return noiseFunction();} ); generator.Forward(std::move(noise)); - discriminator.predictors.col(numFunctions) = boost::apply_visitor( - outputParameterVisitor, generator.network.back());; - discriminator.Forward(std::move(discriminator.predictors.col(numFunctions))); - discriminator.responses(numFunctions) = 0; + discriminator.predictors.cols(numFunctions, numFunctions + batchSize - 1) = + boost::apply_visitor(outputParameterVisitor, generator.network.back()); + discriminator.Forward(std::move(discriminator.predictors.cols(numFunctions, + numFunctions + batchSize - 1))); + discriminator.responses.cols(numFunctions, numFunctions + batchSize - 1) = + arma::zeros(1, batchSize); - currentTarget = discriminator.responses.unsafe_col(numFunctions); + currentTarget = arma::mat(discriminator.responses.memptr() + numFunctions, + 1, batchSize, false, false); res += discriminator.outputLayer.Forward( std::move(boost::apply_visitor( outputParameterVisitor, @@ -190,13 +197,13 @@ Gradient(const arma::mat& /*parameters*/, const size_t i, arma::mat& gradient, // Get the gradients of the Discriminator. discriminator.Gradient(discriminator.parameter, i, gradientDiscriminator, batchSize); - noise.imbue( [&]() { return noiseFunction();} ); generator.Forward(std::move(noise)); - discriminator.predictors.col(numFunctions) = boost::apply_visitor( - outputParameterVisitor, generator.network.back()); + discriminator.predictors.cols(numFunctions, numFunctions + batchSize - 1) = + boost::apply_visitor(outputParameterVisitor, generator.network.back()); - discriminator.responses(numFunctions) = 0; + discriminator.responses.cols(numFunctions, numFunctions + batchSize - 1) = + arma::zeros(1, batchSize); discriminator.Gradient(discriminator.parameter, numFunctions, noiseGradientDiscriminator, batchSize); gradientDiscriminator += noiseGradientDiscriminator; @@ -205,7 +212,8 @@ Gradient(const arma::mat& /*parameters*/, const size_t i, arma::mat& gradient, { // Minimize -log(D(G(noise))). // Pass the error from Discriminator to Generator. - discriminator.responses(numFunctions) = 1; + discriminator.responses.cols(numFunctions, numFunctions + batchSize - 1) = + arma::ones(1, batchSize); discriminator.Gradient(discriminator.parameter, numFunctions, noiseGradientDiscriminator, batchSize); generator.error = boost::apply_visitor(deltaVisitor, @@ -213,7 +221,7 @@ Gradient(const arma::mat& /*parameters*/, const size_t i, arma::mat& gradient, generator.Predictors() = noise; generator.ResetGradients(gradientGenerator); - generator.Gradient(generator.parameter, 0, gradientGenerator, noise.n_cols); + generator.Gradient(generator.parameter, 0, gradientGenerator, batchSize); gradientGenerator *= multiplier; } diff --git a/src/mlpack/methods/ann/layer/atrous_convolution.hpp b/src/mlpack/methods/ann/layer/atrous_convolution.hpp index 79bf1def2d..18aefbfe18 100644 --- a/src/mlpack/methods/ann/layer/atrous_convolution.hpp +++ b/src/mlpack/methods/ann/layer/atrous_convolution.hpp @@ -272,12 +272,15 @@ class AtrousConvolution } } - //! Locally-stored number of input units. + //! Locally-stored number of input channels. size_t inSize; - //! Locally-stored number of output units. + //! Locally-stored number of output channels. size_t outSize; + //! Locally-stored number of input units. + size_t batchSize; + //! Locally-stored filter/kernel width. size_t kW; diff --git a/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp b/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp index 5593a96f81..519a409760 100644 --- a/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp @@ -118,7 +118,9 @@ void AtrousConvolution< OutputDataType >::Forward(const arma::Mat&& input, arma::Mat&& output) { - inputTemp = arma::cube(input.memptr(), inputWidth, inputHeight, inSize); + batchSize = input.n_cols; + inputTemp = arma::cube(const_cast&&>(input).memptr(), + inputWidth, inputHeight, inSize * batchSize, false, false); if (padW != 0 || padH != 0) { @@ -128,30 +130,41 @@ void AtrousConvolution< size_t wConv = ConvOutSize(inputWidth, kW, dW, padW, dilationW); size_t hConv = ConvOutSize(inputHeight, kH, dH, padH, dilationH); - output.set_size(wConv * hConv * outSize, 1); - outputTemp = arma::Cube(output.memptr(), wConv, hConv, outSize, - false, false); + output.set_size(wConv * hConv * outSize, batchSize); + outputTemp = arma::Cube(output.memptr(), wConv, hConv, + outSize * batchSize, false, false); outputTemp.zeros(); - for (size_t outMap = 0, outMapIdx = 0; outMap < outSize; outMap++) + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) { + if (outMap != 0 && outMap % outSize == 0) + { + batchCount++; + outMapIdx = 0; + } + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) { arma::Mat convOutput; + if (padW != 0 || padH != 0) { - ForwardConvolutionRule::Convolution(inputPaddedTemp.slice(inMap), - weight.slice(outMapIdx), convOutput, dW, dH, dilationW, dilationH); + ForwardConvolutionRule::Convolution(inputPaddedTemp.slice(inMap + + batchCount * inSize), weight.slice(outMapIdx), convOutput, dW, dH, + dilationW, dilationH); } else { - ForwardConvolutionRule::Convolution(inputTemp.slice(inMap), - weight.slice(outMapIdx), convOutput, dW, dH, dilationW, dilationH); + ForwardConvolutionRule::Convolution(inputTemp.slice(inMap + + batchCount * inSize), weight.slice(outMapIdx), convOutput, dW, dH, + dilationW, dilationH); } + outputTemp.slice(outMap) += convOutput; } - outputTemp.slice(outMap) += bias(outMap); + outputTemp.slice(outMap) += bias(outMap % outSize); } outputWidth = outputTemp.n_rows; @@ -175,16 +188,23 @@ void AtrousConvolution< >::Backward( const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { - arma::cube mappedError(gy.memptr(), outputWidth, outputHeight, outSize, - false, false); + arma::cube mappedError(gy.memptr(), outputWidth, outputHeight, + outSize * batchSize, false, false); - g.set_size(inputTemp.n_rows * inputTemp.n_cols * inputTemp.n_slices, 1); + g.set_size(inputTemp.n_rows * inputTemp.n_cols * inSize, batchSize); gTemp = arma::Cube(g.memptr(), inputTemp.n_rows, inputTemp.n_cols, inputTemp.n_slices, false, false); gTemp.zeros(); - for (size_t outMap = 0, outMapIdx = 0; outMap < outSize; outMap++) + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) { + if (outMap != 0 && outMap % outSize == 0) + { + batchCount++; + outMapIdx = 0; + } + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) { arma::Mat output, rotatedFilter; @@ -195,14 +215,14 @@ void AtrousConvolution< if (padW != 0 || padH != 0) { - gTemp.slice(inMap) += output.submat(rotatedFilter.n_rows / 2, - rotatedFilter.n_cols / 2, + gTemp.slice(inMap + batchCount * inSize) += output.submat( + rotatedFilter.n_rows / 2, rotatedFilter.n_cols / 2, rotatedFilter.n_rows / 2 + gTemp.n_rows - 1, rotatedFilter.n_cols / 2 + gTemp.n_cols - 1); } else { - gTemp.slice(inMap) += output; + gTemp.slice(inMap + batchCount * inSize) += output; } } } @@ -231,81 +251,76 @@ void AtrousConvolution< if (padW != 0 && padH != 0) { mappedError = arma::cube(error.memptr(), outputWidth / padW, - outputHeight / padH, outSize); + outputHeight / padH, outSize * batchSize, false, false); } else { mappedError = arma::cube(error.memptr(), outputWidth, - outputHeight, outSize); + outputHeight, outSize * batchSize, false, false); } gradient.set_size(weights.n_elem, 1); - gradientTemp = arma::Cube(gradient.memptr(), weight.n_rows, weight.n_cols, - weight.n_slices, false, false); + gradientTemp = arma::Cube(gradient.memptr(), weight.n_rows, + weight.n_cols, weight.n_slices, false, false); gradientTemp.zeros(); - for (size_t outMap = 0, outMapIdx = 0; outMap < outSize; outMap++) + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) { - for (size_t inMap = 0, s = outMap; inMap < inSize; inMap++, outMapIdx++, - s += outSize) + if (outMap != 0 && outMap % outSize == 0) { - arma::Cube inputSlices; + batchCount++; + outMapIdx = 0; + } + + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) + { + arma::Mat inputSlice; if (padW != 0 || padH != 0) { - inputSlices = inputPaddedTemp.slices(inMap, inMap); + inputSlice = inputPaddedTemp.slice(inMap + batchCount * inSize); } else { - inputSlices = inputTemp.slices(inMap, inMap); + inputSlice = inputTemp.slice(inMap + batchCount * inSize); } - arma::Cube deltaSlices = mappedError.slices(outMap, outMap); + arma::Mat deltaSlice = mappedError.slice(outMap); - arma::Cube output, reducedOutput; - GradientConvolutionRule::Convolution(inputSlices, deltaSlices, + arma::Mat output; + GradientConvolutionRule::Convolution(inputSlice, deltaSlice, output, dW, dH, 1, 1); - arma::Mat reducedMat; - reducedOutput = arma::zeros >(kW, kH, output.n_slices); - for (size_t j = 0; j < output.n_slices; j++) + if (dilationH > 1) { - reducedMat = output.slice(j); - if (dilationH > 1) - { - for (size_t i = 1; i < reducedMat.n_cols; i++){ - reducedMat.shed_cols(i, i + dilationH - 2); - } + for (size_t i = 1; i < output.n_cols; i++){ + output.shed_cols(i, i + dilationH - 2); } - if (dilationW > 1) - { - for (size_t i = 1; i < reducedMat.n_rows; i++){ - reducedMat.shed_rows(i, i + dilationW - 2); - } + } + if (dilationW > 1) + { + for (size_t i = 1; i < output.n_rows; i++){ + output.shed_rows(i, i + dilationW - 2); } - reducedOutput.slice(j) = reducedMat; } if ((padW != 0 || padH != 0) && - (gradientTemp.n_rows < reducedOutput.n_rows && - gradientTemp.n_cols < reducedOutput.n_cols)) + (gradientTemp.n_rows < output.n_rows && + gradientTemp.n_cols < output.n_cols)) { - for (size_t i = 0; i < reducedOutput.n_slices; i++) - { - gradientTemp.slice(s) += reducedOutput.slice(i).submat( - reducedOutput.n_rows / 2, reducedOutput.n_cols / 2, - reducedOutput.n_rows / 2 + gradientTemp.n_rows - 1, - reducedOutput.n_cols / 2 + gradientTemp.n_cols - 1); - } + gradientTemp.slice(outMapIdx) += output.submat(output.n_rows / 2, + output.n_cols / 2, + output.n_rows / 2 + gradientTemp.n_rows - 1, + output.n_cols / 2 + gradientTemp.n_cols - 1); } else { - for (size_t i = 0; i < reducedOutput.n_slices; i++) - gradientTemp.slice(s) += reducedOutput.slice(i); + gradientTemp.slice(outMapIdx) += output; } } - gradient.submat(weight.n_elem + outMap, 0, weight.n_elem + outMap, 0) = - arma::accu(mappedError.slices(outMap, outMap)); + gradient.submat(weight.n_elem + (outMap % outSize), 0, weight.n_elem + + (outMap % outSize), 0) = arma::accu(mappedError.slice(outMap)); } } @@ -328,6 +343,7 @@ void AtrousConvolution< { ar & BOOST_SERIALIZATION_NVP(inSize); ar & BOOST_SERIALIZATION_NVP(outSize); + ar & BOOST_SERIALIZATION_NVP(batchSize); ar & BOOST_SERIALIZATION_NVP(kW); ar & BOOST_SERIALIZATION_NVP(kH); ar & BOOST_SERIALIZATION_NVP(dW); diff --git a/src/mlpack/methods/ann/layer/bilinear_interpolation.hpp b/src/mlpack/methods/ann/layer/bilinear_interpolation.hpp index 11305648b9..385b2bc4a3 100644 --- a/src/mlpack/methods/ann/layer/bilinear_interpolation.hpp +++ b/src/mlpack/methods/ann/layer/bilinear_interpolation.hpp @@ -1,6 +1,7 @@ /** * @file bilinear_interpolation.hpp - * @author Kris Singh and Shikhar Jaiswal + * @author Kris Singh + * @author Shikhar Jaiswal * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -114,6 +115,8 @@ class BilinearInterpolation size_t outColSize; //! Locally stored depth of the input. size_t depth; + //! Locally stored number of input points. + size_t batchSize; //! Locally-stored delta object. OutputDataType delta; //! Locally-stored input parameter object. diff --git a/src/mlpack/methods/ann/layer/bilinear_interpolation_impl.hpp b/src/mlpack/methods/ann/layer/bilinear_interpolation_impl.hpp index b9e728b5e6..bd22e2684a 100644 --- a/src/mlpack/methods/ann/layer/bilinear_interpolation_impl.hpp +++ b/src/mlpack/methods/ann/layer/bilinear_interpolation_impl.hpp @@ -1,6 +1,7 @@ /** * @file bilinear_interpolation_impl.hpp - * @author Kris Singh and Shikhar Jaiswal + * @author Kris Singh + * @author Shikhar Jaiswal * * Implementation of the bilinear interpolation function as an individual layer. * @@ -26,7 +27,8 @@ BilinearInterpolation(): inColSize(0), outRowSize(0), outColSize(0), - depth(0) + depth(0), + batchSize(0) { // Nothing to do here. } @@ -43,7 +45,8 @@ BilinearInterpolation( inColSize(inColSize), outRowSize(outRowSize), outColSize(outColSize), - depth(depth) + depth(depth), + batchSize(0) { // Nothing to do here. } @@ -53,20 +56,22 @@ template void BilinearInterpolation::Forward( const arma::Mat&& input, arma::Mat&& output) { + batchSize = input.n_cols; if (output.is_empty()) - output.set_size(outRowSize * outColSize * depth, 1); + output.set_size(outRowSize * outColSize * depth, batchSize); else { assert(output.n_rows == outRowSize * outColSize * depth); - assert(output.n_cols == 1); + assert(output.n_cols == batchSize); } assert(inRowSize >= 2); assert(inColSize >= 2); - arma::cube inputAsCube(input.memptr(), inRowSize, inColSize, depth); - arma::cube outputAsCube(output.memptr(), outRowSize, outColSize, depth, - false, true); + arma::cube inputAsCube(const_cast&&>(input).memptr(), + inRowSize, inColSize, depth * batchSize, false, false); + arma::cube outputAsCube(output.memptr(), outRowSize, outColSize, + depth * batchSize, false, true); double scaleRow = (double) inRowSize / (double) outRowSize; double scaleCol = (double) inColSize / (double) outColSize; @@ -97,7 +102,7 @@ void BilinearInterpolation::Forward( coeffs[2] = (1 - deltaR) * deltaC; coeffs[3] = deltaR * deltaC; - for (size_t k = 0; k < depth; k++) + for (size_t k = 0; k < depth * batchSize; k++) { outputAsCube(i, j, k) = arma::accu(inputAsCube.slice(k).submat( rOrigin, cOrigin, rOrigin + 1, cOrigin + 1) % coeffs); @@ -114,19 +119,20 @@ void BilinearInterpolation::Backward( arma::Mat&& output) { if (output.is_empty()) - output.set_size(inRowSize * inColSize * depth, 1); + output.set_size(inRowSize * inColSize * depth, batchSize); else { assert(output.n_rows == inRowSize * inColSize * depth); - assert(output.n_cols == 1); + assert(output.n_cols == batchSize); } assert(outRowSize >= 2); assert(outColSize >= 2); - arma::cube gradientAsCube(gradient.memptr(), outRowSize, outColSize, depth); - arma::cube outputAsCube(output.memptr(), inRowSize, inColSize, depth, false, - true); + arma::cube gradientAsCube(gradient.memptr(), outRowSize, outColSize, + depth * batchSize, false, false); + arma::cube outputAsCube(output.memptr(), inRowSize, inColSize, + depth * batchSize, false, true); if (gradient.n_elem == output.n_elem) { @@ -157,7 +163,7 @@ void BilinearInterpolation::Backward( coeffs[2] = (1 - deltaR) * deltaC; coeffs[3] = deltaR * deltaC; - for (size_t k = 0; k < depth; k++) + for (size_t k = 0; k < depth * batchSize; k++) { outputAsCube(i, j, k) = arma::accu(gradientAsCube.slice(k).submat( rOrigin, cOrigin, rOrigin + 1, cOrigin + 1) % coeffs); diff --git a/src/mlpack/methods/ann/layer/convolution.hpp b/src/mlpack/methods/ann/layer/convolution.hpp index 940581c138..83c2e09457 100644 --- a/src/mlpack/methods/ann/layer/convolution.hpp +++ b/src/mlpack/methods/ann/layer/convolution.hpp @@ -261,12 +261,15 @@ class Convolution } } - //! Locally-stored number of input units. + //! Locally-stored number of input channels. size_t inSize; - //! Locally-stored number of output units. + //! Locally-stored number of output channels. size_t outSize; + //! Locally-stored number of input units. + size_t batchSize; + //! Locally-stored filter/kernel width. size_t kW; diff --git a/src/mlpack/methods/ann/layer/convolution_impl.hpp b/src/mlpack/methods/ann/layer/convolution_impl.hpp index 0b2618c860..1eba6478a7 100644 --- a/src/mlpack/methods/ann/layer/convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/convolution_impl.hpp @@ -113,7 +113,9 @@ void Convolution< OutputDataType >::Forward(const arma::Mat&& input, arma::Mat&& output) { - inputTemp = arma::cube(input.memptr(), inputWidth, inputHeight, inSize); + batchSize = input.n_cols; + inputTemp = arma::cube(const_cast&&>(input).memptr(), + inputWidth, inputHeight, inSize * batchSize, false, false); if (padW != 0 || padH != 0) { @@ -123,32 +125,39 @@ void Convolution< size_t wConv = ConvOutSize(inputWidth, kW, dW, padW); size_t hConv = ConvOutSize(inputHeight, kH, dH, padH); - output.set_size(wConv * hConv * outSize, 1); - outputTemp = arma::Cube(output.memptr(), wConv, hConv, outSize, - false, false); + output.set_size(wConv * hConv * outSize, batchSize); + outputTemp = arma::Cube(output.memptr(), wConv, hConv, + outSize * batchSize, false, false); outputTemp.zeros(); - for (size_t outMap = 0, outMapIdx = 0; outMap < outSize; outMap++) + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) { + if (outMap != 0 && outMap % outSize == 0) + { + batchCount++; + outMapIdx = 0; + } + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) { arma::Mat convOutput; if (padW != 0 || padH != 0) { - ForwardConvolutionRule::Convolution(inputPaddedTemp.slice(inMap), - weight.slice(outMapIdx), convOutput, dW, dH); + ForwardConvolutionRule::Convolution(inputPaddedTemp.slice(inMap + + batchCount * inSize), weight.slice(outMapIdx), convOutput, dW, dH); } else { - ForwardConvolutionRule::Convolution(inputTemp.slice(inMap), - weight.slice(outMapIdx), convOutput, dW, dH); + ForwardConvolutionRule::Convolution(inputTemp.slice(inMap + + batchCount * inSize), weight.slice(outMapIdx), convOutput, dW, dH); } outputTemp.slice(outMap) += convOutput; } - outputTemp.slice(outMap) += bias(outMap); + outputTemp.slice(outMap) += bias(outMap % outSize); } outputWidth = outputTemp.n_rows; @@ -172,35 +181,41 @@ void Convolution< >::Backward( const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { - arma::cube mappedError(gy.memptr(), outputWidth, outputHeight, outSize, - false, false); + arma::cube mappedError(gy.memptr(), outputWidth, outputHeight, + outSize * batchSize, false, false); - g.set_size(inputTemp.n_rows * inputTemp.n_cols * inputTemp.n_slices, 1); + g.set_size(inputTemp.n_rows * inputTemp.n_cols * inSize, batchSize); gTemp = arma::Cube(g.memptr(), inputTemp.n_rows, inputTemp.n_cols, inputTemp.n_slices, false, false); gTemp.zeros(); - for (size_t outMap = 0, outMapIdx = 0; outMap < outSize; outMap++) + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) { + if (outMap != 0 && outMap % outSize == 0) + { + batchCount++; + outMapIdx = 0; + } + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) { - arma::Mat rotatedFilter; + arma::Mat output, rotatedFilter; Rotate180(weight.slice(outMapIdx), rotatedFilter); - arma::Mat output; BackwardConvolutionRule::Convolution(mappedError.slice(outMap), rotatedFilter, output, dW, dH); if (padW != 0 || padH != 0) { - gTemp.slice(inMap) += output.submat(rotatedFilter.n_rows / 2, - rotatedFilter.n_cols / 2, + gTemp.slice(inMap + batchCount * inSize) += output.submat( + rotatedFilter.n_rows / 2, rotatedFilter.n_cols / 2, rotatedFilter.n_rows / 2 + gTemp.n_rows - 1, rotatedFilter.n_cols / 2 + gTemp.n_cols - 1); } else { - gTemp.slice(inMap) += output; + gTemp.slice(inMap + batchCount * inSize) += output; } } } @@ -229,61 +244,63 @@ void Convolution< if (padW != 0 && padH != 0) { mappedError = arma::cube(error.memptr(), outputWidth / padW, - outputHeight / padH, outSize); + outputHeight / padH, outSize * batchSize, false, false); } else { mappedError = arma::cube(error.memptr(), outputWidth, - outputHeight, outSize); + outputHeight, outSize * batchSize, false, false); } gradient.set_size(weights.n_elem, 1); - gradientTemp = arma::Cube(gradient.memptr(), weight.n_rows, weight.n_cols, - weight.n_slices, false, false); + gradientTemp = arma::Cube(gradient.memptr(), weight.n_rows, + weight.n_cols, weight.n_slices, false, false); gradientTemp.zeros(); - for (size_t outMap = 0, outMapIdx = 0; outMap < outSize; outMap++) + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) { - for (size_t inMap = 0, s = outMap; inMap < inSize; inMap++, outMapIdx++, - s += outSize) + if (outMap != 0 && outMap % outSize == 0) { - arma::Cube inputSlices; + batchCount++; + outMapIdx = 0; + } + + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) + { + arma::Mat inputSlice; if (padW != 0 || padH != 0) { - inputSlices = inputPaddedTemp.slices(inMap, inMap); + inputSlice = inputPaddedTemp.slice(inMap + batchCount * inSize); } else { - inputSlices = inputTemp.slices(inMap, inMap); + inputSlice = inputTemp.slice(inMap + batchCount * inSize); } - arma::Cube deltaSlices = mappedError.slices(outMap, outMap); + arma::Mat deltaSlice = mappedError.slice(outMap); - arma::Cube output; - GradientConvolutionRule::Convolution(inputSlices, deltaSlices, + arma::Mat output; + GradientConvolutionRule::Convolution(inputSlice, deltaSlice, output, dW, dH); if ((padW != 0 || padH != 0) && (gradientTemp.n_rows < output.n_rows && gradientTemp.n_cols < output.n_cols)) { - for (size_t i = 0; i < output.n_slices; i++) - { - gradientTemp.slice(s) += output.slice(i).submat(output.n_rows / 2, - output.n_cols / 2, - output.n_rows / 2 + gradientTemp.n_rows - 1, - output.n_cols / 2 + gradientTemp.n_cols - 1); - } + gradientTemp.slice(outMapIdx) += output.submat(output.n_rows / 2, + output.n_cols / 2, + output.n_rows / 2 + gradientTemp.n_rows - 1, + output.n_cols / 2 + gradientTemp.n_cols - 1); } else { - for (size_t i = 0; i < output.n_slices; i++) - gradientTemp.slice(s) += output.slice(i); + gradientTemp.slice(outMapIdx) += output; } } - gradient.submat(weight.n_elem + outMap, 0, weight.n_elem + outMap, 0) = - arma::accu(mappedError.slices(outMap, outMap)); + gradient.submat(weight.n_elem + (outMap % outSize), 0, weight.n_elem + + (outMap % outSize), 0) = arma::accu(mappedError.slice(outMap)); } } @@ -306,6 +323,7 @@ void Convolution< { ar & BOOST_SERIALIZATION_NVP(inSize); ar & BOOST_SERIALIZATION_NVP(outSize); + ar & BOOST_SERIALIZATION_NVP(batchSize); ar & BOOST_SERIALIZATION_NVP(kW); ar & BOOST_SERIALIZATION_NVP(kH); ar & BOOST_SERIALIZATION_NVP(dW); diff --git a/src/mlpack/methods/ann/layer/max_pooling.hpp b/src/mlpack/methods/ann/layer/max_pooling.hpp index 0001819959..fb86c98c89 100644 --- a/src/mlpack/methods/ann/layer/max_pooling.hpp +++ b/src/mlpack/methods/ann/layer/max_pooling.hpp @@ -193,12 +193,15 @@ class MaxPooling } } - //! Locally-stored number of input units. + //! Locally-stored number of input channels. size_t inSize; - //! Locally-stored number of output units. + //! Locally-stored number of output channels. size_t outSize; + //! Locally-stored number of input units. + size_t batchSize; + //! Locally-stored width of the pooling window. size_t kW; diff --git a/src/mlpack/methods/ann/layer/max_pooling_impl.hpp b/src/mlpack/methods/ann/layer/max_pooling_impl.hpp index 13ff201c51..ff532f1afa 100644 --- a/src/mlpack/methods/ann/layer/max_pooling_impl.hpp +++ b/src/mlpack/methods/ann/layer/max_pooling_impl.hpp @@ -43,6 +43,9 @@ MaxPooling::MaxPooling( inputHeight(0), outputWidth(0), outputHeight(0), + batchSize(0), + inSize(0), + outSize(0), deterministic(false) { // Nothing to do here. @@ -53,8 +56,10 @@ template void MaxPooling::Forward( const arma::Mat&& input, arma::Mat&& output) { - const size_t slices = input.n_elem / (inputWidth * inputHeight); - inputTemp = arma::cube(input.memptr(), inputWidth, inputHeight, slices); + batchSize = input.n_cols; + inSize = input.n_elem / (inputWidth * inputHeight * batchSize); + inputTemp = arma::cube(const_cast&&>(input).memptr(), + inputWidth, inputHeight, batchSize * inSize, false, false); if (floor) { @@ -70,7 +75,7 @@ void MaxPooling::Forward( } outputTemp = arma::zeros >(outputWidth, outputHeight, - slices); + batchSize * inSize); if (!deterministic) { @@ -102,11 +107,12 @@ void MaxPooling::Forward( } } - output = arma::Mat(outputTemp.memptr(), outputTemp.n_elem, 1); + output = arma::Mat(outputTemp.memptr(), outputTemp.n_elem / batchSize, + batchSize); outputWidth = outputTemp.n_rows; outputHeight = outputTemp.n_cols; - outSize = slices; + outSize = batchSize * inSize; } template @@ -115,7 +121,7 @@ void MaxPooling::Backward( const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { arma::cube mappedError = arma::cube(gy.memptr(), outputWidth, - outputHeight, outSize); + outputHeight, outSize, false, false); gTemp = arma::zeros(inputTemp.n_rows, inputTemp.n_cols, inputTemp.n_slices); @@ -128,7 +134,7 @@ void MaxPooling::Backward( poolingIndices.pop_back(); - g = arma::mat(gTemp.memptr(), gTemp.n_elem, 1); + g = arma::mat(gTemp.memptr(), gTemp.n_elem / batchSize, batchSize); } template @@ -141,6 +147,7 @@ void MaxPooling::serialize( ar & BOOST_SERIALIZATION_NVP(kH); ar & BOOST_SERIALIZATION_NVP(dW); ar & BOOST_SERIALIZATION_NVP(dH); + ar & BOOST_SERIALIZATION_NVP(batchSize); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/mean_pooling.hpp b/src/mlpack/methods/ann/layer/mean_pooling.hpp index 3c77d30964..d2527512e4 100644 --- a/src/mlpack/methods/ann/layer/mean_pooling.hpp +++ b/src/mlpack/methods/ann/layer/mean_pooling.hpp @@ -177,12 +177,15 @@ class MeanPooling } } - //! Locally-stored number of input units. + //! Locally-stored number of input channels. size_t inSize; - //! Locally-stored number of output units. + //! Locally-stored number of output channels. size_t outSize; + //! Locally-stored number of input units. + size_t batchSize; + //! Locally-stored width of the pooling window. size_t kW; diff --git a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp index 900843e29e..52b2f807df 100644 --- a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp +++ b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp @@ -43,7 +43,10 @@ MeanPooling::MeanPooling( reset(false), floor(floor), deterministic(false), - offset(0) + offset(0), + batchSize(0), + inSize(0), + outSize(0) { // Nothing to do here. } @@ -53,8 +56,10 @@ template void MeanPooling::Forward( const arma::Mat&& input, arma::Mat&& output) { - size_t slices = input.n_elem / (inputWidth * inputHeight); - inputTemp = arma::cube(input.memptr(), inputWidth, inputHeight, slices); + batchSize = input.n_cols; + inSize = input.n_elem / (inputWidth * inputHeight * batchSize); + inputTemp = arma::cube(const_cast&&>(input).memptr(), + inputWidth, inputHeight, batchSize * inSize, false, false); if (floor) { @@ -72,16 +77,17 @@ void MeanPooling::Forward( } outputTemp = arma::zeros >(outputWidth, outputHeight, - slices); + batchSize * inSize); for (size_t s = 0; s < inputTemp.n_slices; s++) Pooling(inputTemp.slice(s), outputTemp.slice(s)); - output = arma::Mat(outputTemp.memptr(), outputTemp.n_elem, 1); + output = arma::Mat(outputTemp.memptr(), outputTemp.n_elem / batchSize, + batchSize); outputWidth = outputTemp.n_rows; outputHeight = outputTemp.n_cols; - outSize = slices; + outSize = batchSize * inSize; } template @@ -92,7 +98,7 @@ void MeanPooling::Backward( arma::Mat&& g) { arma::cube mappedError = arma::cube(gy.memptr(), outputWidth, - outputHeight, outSize); + outputHeight, outSize, false, false); gTemp = arma::zeros(inputTemp.n_rows, inputTemp.n_cols, inputTemp.n_slices); @@ -102,7 +108,7 @@ void MeanPooling::Backward( Unpooling(inputTemp.slice(s), mappedError.slice(s), gTemp.slice(s)); } - g = arma::mat(gTemp.memptr(), gTemp.n_elem, 1); + g = arma::mat(gTemp.memptr(), gTemp.n_elem / batchSize, batchSize); } template @@ -115,6 +121,7 @@ void MeanPooling::serialize( ar & BOOST_SERIALIZATION_NVP(kH); ar & BOOST_SERIALIZATION_NVP(dW); ar & BOOST_SERIALIZATION_NVP(dH); + ar & BOOST_SERIALIZATION_NVP(batchSize); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/transposed_convolution.hpp b/src/mlpack/methods/ann/layer/transposed_convolution.hpp index fc4b20f07d..8df31aa491 100644 --- a/src/mlpack/methods/ann/layer/transposed_convolution.hpp +++ b/src/mlpack/methods/ann/layer/transposed_convolution.hpp @@ -263,12 +263,15 @@ class TransposedConvolution } } - //! Locally-stored number of input units. + //! Locally-stored number of input channels. size_t inSize; - //! Locally-stored number of output units. + //! Locally-stored number of output channels. size_t outSize; + //! Locally-stored number of input units. + size_t batchSize; + //! Locally-stored filter/kernel width. size_t kW; diff --git a/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp b/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp index b667cfbc53..3b0b356dbe 100644 --- a/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp @@ -114,30 +114,39 @@ void TransposedConvolution< OutputDataType >::Forward(const arma::Mat&& input, arma::Mat&& output) { - inputTemp = arma::cube(input.memptr(), inputWidth, inputHeight, inSize); + batchSize = input.n_cols; + inputTemp = arma::cube(const_cast&&>(input).memptr(), + inputWidth, inputHeight, inSize * batchSize, false, false); outputWidth = TransposedConvOutSize(inputWidth, kW, dW, padW); outputHeight = TransposedConvOutSize(inputHeight, kH, dH, padH); - output.set_size(outputWidth * outputHeight * outSize, 1); + output.set_size(outputWidth * outputHeight * outSize, batchSize); outputTemp = arma::Cube(output.memptr(), outputWidth, outputHeight, - outSize, false, false); + outSize * batchSize, false, false); outputTemp.zeros(); - for (size_t outMap = 0, outMapIdx = 0; outMap < outSize; outMap++) + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) { + if (outMap != 0 && outMap % outSize == 0) + { + batchCount++; + outMapIdx = 0; + } + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) { arma::Mat convOutput, rotatedFilter; Rotate180(weight.slice(outMapIdx), rotatedFilter); - BackwardConvolutionRule::Convolution(inputTemp.slice(inMap), - rotatedFilter, convOutput, 1, 1); + BackwardConvolutionRule::Convolution(inputTemp.slice(inMap + + batchCount * inSize), rotatedFilter, convOutput, 1, 1); outputTemp.slice(outMap) += convOutput; } - outputTemp.slice(outMap) += bias(outMap); + outputTemp.slice(outMap) += bias(outMap % outSize); } } @@ -158,16 +167,23 @@ void TransposedConvolution< >::Backward( const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { - arma::cube mappedError(gy.memptr(), outputWidth, outputHeight, outSize, - false, false); - - g.set_size(inputTemp.n_rows * inputTemp.n_cols * inputTemp.n_slices, 1); + arma::cube mappedError(gy.memptr(), outputWidth, outputHeight, + outSize * batchSize, false, false); + g.set_size(inputTemp.n_rows * inputTemp.n_cols * inSize, batchSize); gTemp = arma::Cube(g.memptr(), inputTemp.n_rows, inputTemp.n_cols, inputTemp.n_slices, false, false); + gTemp.zeros(); - for (size_t outMap = 0, outMapIdx = 0; outMap < outSize; outMap++) + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) { + if (outMap != 0 && outMap % outSize == 0) + { + batchCount++; + outMapIdx = 0; + } + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) { arma::Mat output; @@ -175,7 +191,7 @@ void TransposedConvolution< ForwardConvolutionRule::Convolution(mappedError.slice(outMap), weight.slice(outMapIdx), output, 1, 1); - gTemp.slice(inMap) += output; + gTemp.slice(inMap + batchCount * inSize) += output; } } } @@ -200,31 +216,36 @@ void TransposedConvolution< arma::Mat&& gradient) { arma::cube mappedError(error.memptr(), outputWidth, - outputHeight, outSize, false, false); + outputHeight, outSize * batchSize, false, false); gradient.set_size(weights.n_elem, 1); - gradientTemp = arma::Cube(gradient.memptr(), weight.n_rows, weight.n_cols, - weight.n_slices, false, false); + gradientTemp = arma::Cube(gradient.memptr(), weight.n_rows, + weight.n_cols, weight.n_slices, false, false); gradientTemp.zeros(); - for (size_t outMap = 0, outMapIdx = 0; outMap < outSize; outMap++) + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) { - for (size_t inMap = 0, s = outMap; inMap < inSize; inMap++, outMapIdx++, - s += outSize) + if (outMap != 0 && outMap % outSize == 0) { - arma::Cube inputSlices, output; - inputSlices = inputTemp.slices(inMap, inMap); - arma::Cube deltaSlices = mappedError.slices(outMap, outMap); - - GradientConvolutionRule::Convolution(deltaSlices, inputSlices, - output, 1, 1); - - for (size_t i = 0; i < output.n_slices; i++) - gradientTemp.slice(s) += output.slice(i); + batchCount++; + outMapIdx = 0; } - gradient.submat(weight.n_elem + outMap, 0, weight.n_elem + outMap, 0) = - arma::accu(mappedError.slices(outMap, outMap)); + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) + { + arma::Mat inputSlice, output; + inputSlice = inputTemp.slice(inMap + batchCount * inSize); + arma::Mat deltaSlice = mappedError.slice(outMap); + + GradientConvolutionRule::Convolution(deltaSlice, inputSlice, + output, 1, 1); + + gradientTemp.slice(outMapIdx) += output; + } + + gradient.submat(weight.n_elem + (outMap % outSize), 0, weight.n_elem + + (outMap % outSize), 0) = arma::accu(mappedError.slices(outMap, outMap)); } } @@ -247,6 +268,7 @@ void TransposedConvolution< { ar & BOOST_SERIALIZATION_NVP(inSize); ar & BOOST_SERIALIZATION_NVP(outSize); + ar & BOOST_SERIALIZATION_NVP(batchSize); ar & BOOST_SERIALIZATION_NVP(kW); ar & BOOST_SERIALIZATION_NVP(kH); ar & BOOST_SERIALIZATION_NVP(dW); diff --git a/src/mlpack/tests/gan_test.cpp b/src/mlpack/tests/gan_test.cpp index fcc9710757..40fc4c12f2 100644 --- a/src/mlpack/tests/gan_test.cpp +++ b/src/mlpack/tests/gan_test.cpp @@ -140,13 +140,13 @@ BOOST_AUTO_TEST_CASE(GANMNISTTest) { size_t dNumKernels = 32; size_t discriminatorPreTrain = 300; - size_t batchSize = 1; + size_t batchSize = 50; size_t noiseDim = 100; size_t generatorUpdateStep = 1; size_t numSamples = 10; double stepSize = 0.0003; double eps = 1e-8; - size_t numEpoches = 10; + size_t numEpoches = 20; double tolerance = 1e-5; int datasetMaxCols = -1; bool shuffle = true; @@ -221,7 +221,7 @@ BOOST_AUTO_TEST_CASE(GANMNISTTest) // Generate samples Log::Info << "Sampling..." << std::endl; - arma::mat noise(noiseDim, 1); + arma::mat noise(noiseDim, batchSize); size_t dim = std::sqrt(trainData.n_rows); arma::mat generatedData(2 * dim, dim * numSamples); From bbc08d317ba03b883fe3fd69e5bac6131b3b3f97 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Thu, 21 Jun 2018 13:55:42 +0200 Subject: [PATCH 73/79] Remove random seed from test cases. --- src/mlpack/tests/bigbatch_sgd_test.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/mlpack/tests/bigbatch_sgd_test.cpp b/src/mlpack/tests/bigbatch_sgd_test.cpp index ca365d61bc..a710563439 100644 --- a/src/mlpack/tests/bigbatch_sgd_test.cpp +++ b/src/mlpack/tests/bigbatch_sgd_test.cpp @@ -84,8 +84,6 @@ void CreateLogisticRegressionTestData(arma::mat& data, */ BOOST_AUTO_TEST_CASE(BBSBBLogisticRegressionTest) { - mlpack::math::RandomSeed(time(NULL)); - arma::mat data, testData, shuffledData; arma::Row responses, testResponses, shuffledResponses; @@ -113,8 +111,6 @@ BOOST_AUTO_TEST_CASE(BBSBBLogisticRegressionTest) */ BOOST_AUTO_TEST_CASE(BBSArmijoLogisticRegressionTest) { - mlpack::math::RandomSeed(time(NULL)); - arma::mat data, testData, shuffledData; arma::Row responses, testResponses, shuffledResponses; From 3b3b2822ca876a40bfe96aec9635404d22c243fc Mon Sep 17 00:00:00 2001 From: akhandait Date: Thu, 21 Jun 2018 01:47:20 +0530 Subject: [PATCH 74/79] fix static code check --- src/mlpack/methods/ann/layer/reparametrization_impl.hpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/ann/layer/reparametrization_impl.hpp b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp index 58264218e6..68e6defdb7 100644 --- a/src/mlpack/methods/ann/layer/reparametrization_impl.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp @@ -20,7 +20,10 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -Reparametrization::Reparametrization() +Reparametrization::Reparametrization() : + latentSize(0), + stochastic(true), + includeKl(true) { // Nothing to do here. } @@ -53,9 +56,9 @@ void Reparametrization::Forward( preStdDev = input.submat(0, 0, latentSize - 1, input.n_cols - 1); if (stochastic) - gaussianSample = arma::randn>(latentSize, input.n_cols); + gaussianSample = arma::randn >(latentSize, input.n_cols); else - gaussianSample = arma::ones>(latentSize, input.n_cols) * 0.7; + gaussianSample = arma::ones >(latentSize, input.n_cols) * 0.7; SoftplusFunction::Fn(preStdDev, stdDev); output = mean + stdDev % gaussianSample; From d2786582722e83851fb645cdc4da2b6b96032f83 Mon Sep 17 00:00:00 2001 From: Shikhar Jaiswal Date: Sat, 2 Jun 2018 00:47:58 +0530 Subject: [PATCH 75/79] Implement DCGAN Test --- .../ann/layer/atrous_convolution_impl.hpp | 14 +- .../methods/ann/layer/convolution_impl.hpp | 14 +- src/mlpack/methods/ann/layer/leaky_relu.hpp | 2 +- src/mlpack/tests/CMakeLists.txt | 1 + src/mlpack/tests/dcgan_test.cpp | 280 ++++++++++++++++++ src/mlpack/tests/gan_test.cpp | 21 +- 6 files changed, 302 insertions(+), 30 deletions(-) create mode 100644 src/mlpack/tests/dcgan_test.cpp diff --git a/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp b/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp index 519a409760..bac305fa33 100644 --- a/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp @@ -215,10 +215,9 @@ void AtrousConvolution< if (padW != 0 || padH != 0) { - gTemp.slice(inMap + batchCount * inSize) += output.submat( - rotatedFilter.n_rows / 2, rotatedFilter.n_cols / 2, - rotatedFilter.n_rows / 2 + gTemp.n_rows - 1, - rotatedFilter.n_cols / 2 + gTemp.n_cols - 1); + gTemp.slice(inMap + batchCount * inSize) += output.submat(padW, padH, + padW + gTemp.n_rows - 1, + padH + gTemp.n_cols - 1); } else { @@ -308,10 +307,9 @@ void AtrousConvolution< (gradientTemp.n_rows < output.n_rows && gradientTemp.n_cols < output.n_cols)) { - gradientTemp.slice(outMapIdx) += output.submat(output.n_rows / 2, - output.n_cols / 2, - output.n_rows / 2 + gradientTemp.n_rows - 1, - output.n_cols / 2 + gradientTemp.n_cols - 1); + gradientTemp.slice(outMapIdx) += output.submat(padW, padH, + padW + gradientTemp.n_rows - 1, + padH + gradientTemp.n_cols - 1); } else { diff --git a/src/mlpack/methods/ann/layer/convolution_impl.hpp b/src/mlpack/methods/ann/layer/convolution_impl.hpp index 1eba6478a7..e06905b845 100644 --- a/src/mlpack/methods/ann/layer/convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/convolution_impl.hpp @@ -208,10 +208,9 @@ void Convolution< if (padW != 0 || padH != 0) { - gTemp.slice(inMap + batchCount * inSize) += output.submat( - rotatedFilter.n_rows / 2, rotatedFilter.n_cols / 2, - rotatedFilter.n_rows / 2 + gTemp.n_rows - 1, - rotatedFilter.n_cols / 2 + gTemp.n_cols - 1); + gTemp.slice(inMap + batchCount * inSize) += output.submat(padW, padH, + padW + gTemp.n_rows - 1, + padH + gTemp.n_cols - 1); } else { @@ -288,10 +287,9 @@ void Convolution< (gradientTemp.n_rows < output.n_rows && gradientTemp.n_cols < output.n_cols)) { - gradientTemp.slice(outMapIdx) += output.submat(output.n_rows / 2, - output.n_cols / 2, - output.n_rows / 2 + gradientTemp.n_rows - 1, - output.n_cols / 2 + gradientTemp.n_cols - 1); + gradientTemp.slice(outMapIdx) += output.submat(padW, padH, + padW + gradientTemp.n_rows - 1, + padH + gradientTemp.n_cols - 1); } else { diff --git a/src/mlpack/methods/ann/layer/leaky_relu.hpp b/src/mlpack/methods/ann/layer/leaky_relu.hpp index dfa7af8e92..a5ef7ed6d8 100644 --- a/src/mlpack/methods/ann/layer/leaky_relu.hpp +++ b/src/mlpack/methods/ann/layer/leaky_relu.hpp @@ -46,7 +46,7 @@ class LeakyReLU public: /** * Create the LeakyReLU object using the specified parameters. - * The non zero gradient can be adjusted by specifying tha parameter + * The non zero gradient can be adjusted by specifying the parameter * alpha in the range 0 to 1. Default (alpha = 0.03) * * @param alpha Non zero gradient diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index bf6dfa75b1..a32e5407d5 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -26,6 +26,7 @@ add_executable(mlpack_test cosine_tree_test.cpp cv_test.cpp dbscan_test.cpp + dcgan_test.cpp decision_stump_test.cpp decision_tree_test.cpp det_test.cpp diff --git a/src/mlpack/tests/dcgan_test.cpp b/src/mlpack/tests/dcgan_test.cpp new file mode 100644 index 0000000000..1fb94714f3 --- /dev/null +++ b/src/mlpack/tests/dcgan_test.cpp @@ -0,0 +1,280 @@ +/** + * @file dcgan_network_test.cpp + * @author Shikhar Jaiswal + * + * Tests the DCGAN network. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "test_tools.hpp" + +using namespace mlpack; +using namespace mlpack::ann; +using namespace mlpack::math; +using namespace mlpack::optimization; +using namespace mlpack::regression; +using namespace std::placeholders; + +BOOST_AUTO_TEST_SUITE(DCGANNetworkTest); + +/* + * Tests the DCGAN implementation on the MNIST dataset. + * It's not viable to train on bigger parameters due to time constraints. + * Please refer mlpack/models repository for the tutorial. + */ +BOOST_AUTO_TEST_CASE(DCGANMNISTTest) +{ + size_t dNumKernels = 32; + size_t discriminatorPreTrain = 5; + size_t batchSize = 5; + size_t noiseDim = 100; + size_t generatorUpdateStep = 1; + size_t numSamples = 10; + double stepSize = 0.0003; + double eps = 1e-8; + size_t numEpoches = 1; + double tolerance = 1e-5; + int datasetMaxCols = 10; + bool shuffle = true; + double multiplier = 10; + + Log::Info << std::boolalpha + << " batchSize = " << batchSize << std::endl + << " generatorUpdateStep = " << generatorUpdateStep << std::endl + << " noiseDim = " << noiseDim << std::endl + << " numSamples = " << numSamples << std::endl + << " stepSize = " << stepSize << std::endl + << " numEpoches = " << numEpoches << std::endl + << " tolerance = " << tolerance << std::endl + << " shuffle = " << shuffle << std::endl; + + arma::mat trainData; + trainData.load("mnist_first250_training_4s_and_9s.arm"); + Log::Info << arma::size(trainData) << std::endl; + + if (datasetMaxCols > 0) + trainData = trainData.cols(0, datasetMaxCols - 1); + + size_t numIterations = trainData.n_cols * numEpoches; + numIterations /= batchSize; + + Log::Info << "Dataset loaded (" << trainData.n_rows << ", " + << trainData.n_cols << ")" << std::endl; + Log::Info << trainData.n_rows << "--------" << trainData.n_cols << std::endl; + + // Create the Discriminator network + FFN > discriminator; + discriminator.Add >(1, dNumKernels, 4, 4, 2, 2, 1, 1, 28, 28); + discriminator.Add >(0.2); + discriminator.Add >(dNumKernels, 2 * dNumKernels, 4, 4, 2, 2, + 1, 1, 14, 14); + discriminator.Add >(0.2); + discriminator.Add >(2 * dNumKernels, 4 * dNumKernels, 4, 4, + 2, 2, 1, 1, 7, 7); + discriminator.Add >(0.2); + discriminator.Add >(4 * dNumKernels, 8 * dNumKernels, 4, 4, + 2, 2, 2, 2, 3, 3); + discriminator.Add >(0.2); + discriminator.Add >(8 * dNumKernels, 1, 4, 4, 1, 1, + 1, 1, 2, 2); + discriminator.Add >(); + + // Create the Generator network + FFN > generator; + generator.Add >(noiseDim, 8 * dNumKernels, 2, 2, + 1, 1, 1, 1, 1, 1); + generator.Add >(); + generator.Add >(8 * dNumKernels, 4 * dNumKernels, + 2, 2, 1, 1, 0, 0, 2, 2); + generator.Add >(); + generator.Add >(4 * dNumKernels, 2 * dNumKernels, + 5, 5, 2, 2, 1, 1, 3, 3); + generator.Add >(); + generator.Add >(2 * dNumKernels, dNumKernels, 8, 8, + 1, 1, 1, 1, 7, 7); + generator.Add >(); + generator.Add >(dNumKernels, 1, 15, 15, 1, 1, 1, 1, + 14, 14); + generator.Add >(); + + // Create GAN + GaussianInitialization gaussian(0, 1); + Adam optimizer(stepSize, batchSize, 0.9, 0.999, eps, numIterations, + tolerance, shuffle); + std::function noiseFunction = [] () { + return math::RandNormal(0, 1);}; + GAN >, GaussianInitialization, + std::function > gan(trainData, generator, discriminator, + gaussian, noiseFunction, noiseDim, batchSize, generatorUpdateStep, + discriminatorPreTrain, multiplier); + + Log::Info << "Training..." << std::endl; + gan.Train(optimizer); + + // Generate samples + Log::Info << "Sampling..." << std::endl; + arma::mat noise(noiseDim, 1); + size_t dim = std::sqrt(trainData.n_rows); + arma::mat generatedData(2 * dim, dim * numSamples); + + for (size_t i = 0; i < numSamples; i++) + { + arma::mat samples; + noise.imbue( [&]() { return noiseFunction(); } ); + + generator.Forward(noise, samples); + samples.reshape(dim, dim); + samples = samples.t(); + + generatedData.submat(0, i * dim, dim - 1, i * dim + dim - 1) = samples; + + samples = trainData.col(math::RandInt(0, trainData.n_cols)); + samples.reshape(dim, dim); + samples = samples.t(); + + generatedData.submat(dim, + i * dim, 2 * dim - 1, i * dim + dim - 1) = samples; + } + + Log::Info << "Output generated!" << std::endl; +} + +/* + * Tests the DCGAN implementation on the CelebA dataset. + * It's currently not possible to run this every time due to time constraints. + * Please refer mlpack/models repository for the tutorial. + +BOOST_AUTO_TEST_CASE(DCGANCelebATest) +{ + size_t dNumKernels = 64; + size_t discriminatorPreTrain = 300; + size_t batchSize = 1; + size_t noiseDim = 100; + size_t generatorUpdateStep = 1; + size_t numSamples = 10; + double stepSize = 0.0003; + double eps = 1e-8; + size_t numEpoches = 20; + double tolerance = 1e-5; + int datasetMaxCols = -1; + bool shuffle = true; + double multiplier = 10; + + Log::Info << std::boolalpha + << " batchSize = " << batchSize << std::endl + << " generatorUpdateStep = " << generatorUpdateStep << std::endl + << " noiseDim = " << noiseDim << std::endl + << " numSamples = " << numSamples << std::endl + << " stepSize = " << stepSize << std::endl + << " numEpoches = " << numEpoches << std::endl + << " tolerance = " << tolerance << std::endl + << " shuffle = " << shuffle << std::endl; + + arma::mat trainData; + trainData.load("celeba.csv"); + Log::Info << arma::size(trainData) << std::endl; + + if (datasetMaxCols > 0) + trainData = trainData.cols(0, datasetMaxCols - 1); + + size_t numIterations = trainData.n_cols * numEpoches; + numIterations /= batchSize; + + Log::Info << "Dataset loaded (" << trainData.n_rows << ", " + << trainData.n_cols << ")" << std::endl; + Log::Info << trainData.n_rows << "--------" << trainData.n_cols << std::endl; + + // Create the Discriminator network + FFN > discriminator; + discriminator.Add >(3, dNumKernels, 4, 4, 2, 2, 1, 1, 64, 64); + discriminator.Add >(0.2); + discriminator.Add >(dNumKernels, 2 * dNumKernels, 4, 4, 2, 2, + 1, 1, 32, 32); + discriminator.Add >(0.2); + discriminator.Add >(2 * dNumKernels, 4 * dNumKernels, 4, 4, + 2, 2, 1, 1, 16, 16); + discriminator.Add >(0.2); + discriminator.Add >(4 * dNumKernels, 8 * dNumKernels, 4, 4, + 2, 2, 1, 1, 8, 8); + discriminator.Add >(0.2); + discriminator.Add >(8 * dNumKernels, 1, 4, 4, 1, 1, + 0, 0, 4, 4); + discriminator.Add >(); + + // Create the Generator network + FFN > generator; + generator.Add >(noiseDim, 8 * dNumKernels, 4, 4, + 1, 1, 2, 2, 1, 1); + generator.Add >(); + generator.Add >(8 * dNumKernels, 4 * dNumKernels, + 5, 5, 1, 1, 1, 1, 4, 4); + generator.Add >(); + generator.Add >(4 * dNumKernels, 2 * dNumKernels, + 9, 9, 1, 1, 1, 1, 8, 8); + generator.Add >(); + generator.Add >(2 * dNumKernels, dNumKernels, 17, 17, + 1, 1, 1, 1, 16, 16); + generator.Add >(); + generator.Add >(dNumKernels, 3, 33, 33, 1, 1, 1, 1, + 32, 32); + generator.Add >(); + + // Create GAN + GaussianInitialization gaussian(0, 1); + Adam optimizer(stepSize, batchSize, 0.9, 0.999, eps, numIterations, + tolerance, shuffle); + std::function noiseFunction = [] () { + return math::RandNormal(0, 1);}; + GAN >, GaussianInitialization, + std::function > gan(trainData, generator, discriminator, + gaussian, noiseFunction, noiseDim, batchSize, generatorUpdateStep, + discriminatorPreTrain, multiplier); + + Log::Info << "Training..." << std::endl; + gan.Train(optimizer); + + // Generate samples + Log::Info << "Sampling..." << std::endl; + arma::mat noise(noiseDim, 1); + size_t dim = std::sqrt(trainData.n_rows); + arma::mat generatedData(2 * dim, dim * numSamples); + + for (size_t i = 0; i < numSamples; i++) + { + arma::mat samples; + noise.imbue( [&]() { return noiseFunction(); } ); + + generator.Forward(noise, samples); + samples.reshape(dim, dim); + samples = samples.t(); + + generatedData.submat(0, i * dim, dim - 1, i * dim + dim - 1) = samples; + + samples = trainData.col(math::RandInt(0, trainData.n_cols)); + samples.reshape(dim, dim); + samples = samples.t(); + + generatedData.submat(dim, + i * dim, 2 * dim - 1, i * dim + dim - 1) = samples; + } + + Log::Info << "Output generated!" << std::endl; +} +*/ + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/gan_test.cpp b/src/mlpack/tests/gan_test.cpp index 40fc4c12f2..a6d47c6f9e 100644 --- a/src/mlpack/tests/gan_test.cpp +++ b/src/mlpack/tests/gan_test.cpp @@ -132,28 +132,26 @@ BOOST_AUTO_TEST_CASE(GANTest) } /* - * Tests the GAN implementation on the O'Reilly Test on the MNIST dataset. - * It's currently not possible to run this every time due to time constraints. + * Tests the GAN implementation of the O'Reilly Test on the MNIST dataset. + * It's not viable to train on bigger parameters due to time constraints. * Please refer mlpack/models repository for the tutorial. - + */ BOOST_AUTO_TEST_CASE(GANMNISTTest) { size_t dNumKernels = 32; - size_t discriminatorPreTrain = 300; - size_t batchSize = 50; + size_t discriminatorPreTrain = 5; + size_t batchSize = 5; size_t noiseDim = 100; size_t generatorUpdateStep = 1; size_t numSamples = 10; double stepSize = 0.0003; double eps = 1e-8; - size_t numEpoches = 20; + size_t numEpoches = 1; double tolerance = 1e-5; - int datasetMaxCols = -1; + int datasetMaxCols = 10; bool shuffle = true; double multiplier = 10; - std::string output_dataset = "output_mnist.csv"; - Log::Info << "output_dataset = '" << output_dataset << "'" << std::endl; Log::Info << std::boolalpha << " batchSize = " << batchSize << std::endl << " generatorUpdateStep = " << generatorUpdateStep << std::endl @@ -244,10 +242,7 @@ BOOST_AUTO_TEST_CASE(GANMNISTTest) i * dim, 2 * dim - 1, i * dim + dim - 1) = samples; } - Log::Info << "Saving output to " << output_dataset << "..." << std::endl; - generatedData.save(output_dataset, arma::csv_ascii); - Log::Info << "Output saved!" << std::endl; + Log::Info << "Output generated!" << std::endl; } -*/ BOOST_AUTO_TEST_SUITE_END(); From c8597a21cf075a1283eeecc5e29b03eea7a59f03 Mon Sep 17 00:00:00 2001 From: akhandait Date: Fri, 22 Jun 2018 16:01:18 +0530 Subject: [PATCH 76/79] change documentation --- src/mlpack/methods/ann/layer/reparametrization.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/ann/layer/reparametrization.hpp b/src/mlpack/methods/ann/layer/reparametrization.hpp index 73ec806e96..ad7b86ae56 100644 --- a/src/mlpack/methods/ann/layer/reparametrization.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization.hpp @@ -44,6 +44,8 @@ class Reparametrization * Create the Reparametrization layer object using the specified sample vector size. * * @param layerSize The number of output units. + * @param stochastic Whether we want random sample or constant. + * @param includeKl Whether we want to include KL loss in backward function. */ Reparametrization(const size_t latentSize, const bool stochastic = true, @@ -88,9 +90,7 @@ class Reparametrization * pass of Kullback–Leibler divergence. Using the results from the * KL divergence feed forward pass. * - * @param input The propagated input activation. - * @param gy The backpropagated error. - * @param g The calculated gradient. + * @param output The calculated gradient of KL divergence. */ template void klBackward(OutputType&& output); From 48a1647cb0800f484b0270024b0780b340486d37 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Fri, 22 Jun 2018 21:37:44 +0200 Subject: [PATCH 77/79] Run test multiple times. --- .../tests/convolutional_network_test.cpp | 83 +++++++++++-------- 1 file changed, 47 insertions(+), 36 deletions(-) diff --git a/src/mlpack/tests/convolutional_network_test.cpp b/src/mlpack/tests/convolutional_network_test.cpp index 831975bb8d..1f7c07078f 100644 --- a/src/mlpack/tests/convolutional_network_test.cpp +++ b/src/mlpack/tests/convolutional_network_test.cpp @@ -74,48 +74,59 @@ BOOST_AUTO_TEST_CASE(VanillaNetworkTest) * | | +-+ | +-+ | +-+ | +-+ | | | * +---+ +---+ +---+ +---+ +---+ +---+ */ - - FFN, RandomInitialization> model; - - model.Add >(1, 8, 5, 5, 1, 1, 0, 0, 28, 28); - model.Add >(); - model.Add >(8, 8, 2, 2); - model.Add >(8, 12, 2, 2); - model.Add >(); - model.Add >(2, 2, 2, 2); - model.Add >(192, 20); - model.Add >(); - model.Add >(20, 10); - model.Add >(); - model.Add >(10, 2); - model.Add >(); - - // Train for only 8 epochs. - RMSProp opt(0.001, 1, 0.88, 1e-8, 8 * nPoints, -1); - - model.Train(X, Y, opt); - - arma::mat predictionTemp; - model.Predict(X, predictionTemp); - arma::mat prediction = arma::zeros(1, predictionTemp.n_cols); - - for (size_t i = 0; i < predictionTemp.n_cols; ++i) + // It isn't guaranteed that the network will converge in the specified number + // of iterations using random weights. If this works 1 of 5 times, I'm fine + // with that. All I want to know is that the network is able to escape from + // local minima and to solve the task. + bool success = false; + for (size_t trial = 0; trial < 5; ++trial) { - prediction(i) = arma::as_scalar(arma::find( - arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1; - } + FFN, RandomInitialization> model; - size_t correct = 0; - for (size_t i = 0; i < X.n_cols; i++) - { - if (prediction(i) == Y(i)) + model.Add >(1, 8, 5, 5, 1, 1, 0, 0, 28, 28); + model.Add >(); + model.Add >(8, 8, 2, 2); + model.Add >(8, 12, 2, 2); + model.Add >(); + model.Add >(2, 2, 2, 2); + model.Add >(192, 20); + model.Add >(); + model.Add >(20, 10); + model.Add >(); + model.Add >(10, 2); + model.Add >(); + + // Train for only 8 epochs. + RMSProp opt(0.001, 1, 0.88, 1e-8, 8 * nPoints, -1); + + model.Train(X, Y, opt); + + arma::mat predictionTemp; + model.Predict(X, predictionTemp); + arma::mat prediction = arma::zeros(1, predictionTemp.n_cols); + + for (size_t i = 0; i < predictionTemp.n_cols; ++i) { - correct++; + prediction(i) = arma::as_scalar(arma::find( + arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1; + } + + size_t correct = 0; + for (size_t i = 0; i < X.n_cols; i++) + { + if (prediction(i) == Y(i)) + correct++; + } + + double classificationError = 1 - double(correct) / X.n_cols; + if (classificationError <= 0.25) + { + success = true; + break; } } - double classificationError = 1 - double(correct) / X.n_cols; - BOOST_REQUIRE_LE(classificationError, 0.25); + BOOST_REQUIRE_EQUAL(success, true); } BOOST_AUTO_TEST_SUITE_END(); From c959f951866c14caf5ba2e45f83d5c9a6a5b225d Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 27 Jun 2018 10:51:53 -0400 Subject: [PATCH 78/79] Specify namespace to handle confused MSVC compiler. --- src/mlpack/methods/radical/radical.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/radical/radical.cpp b/src/mlpack/methods/radical/radical.cpp index a705e61235..2204bd197c 100644 --- a/src/mlpack/methods/radical/radical.cpp +++ b/src/mlpack/methods/radical/radical.cpp @@ -187,7 +187,7 @@ void mlpack::radical::WhitenFeatureMajorMatrix(const mat& matX, { mat matU, matV; vec s; - svd(matU, s, matV, cov(matX)); + arma::svd(matU, s, matV, cov(matX)); matWhitening = matU * diagmat(1 / sqrt(s)) * trans(matV); matXWhitened = matX * matWhitening; } From 308acf31976bb3619f71a1cc0fc7f7cd6e02bb36 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 27 Jun 2018 10:53:16 -0400 Subject: [PATCH 79/79] Update history. --- HISTORY.md | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.md b/HISTORY.md index e930dba62f..fc580bf68f 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,6 @@ ### mlpack ?.?.? ###### ????-??-?? + * Fix Visual Studio compilation issue (#1443). ### mlpack 3.0.2 ###### 2018-06-08