From 5687d8a9746c73f9a4ad61aa15852c183be981d8 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Mon, 6 Feb 2023 10:46:47 +0530 Subject: [PATCH 01/88] Update codes and improve cf doc --- doc/tutorials/cf.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/tutorials/cf.md b/doc/tutorials/cf.md index 9452ebfdd0..2efb086e9d 100644 --- a/doc/tutorials/cf.md +++ b/doc/tutorials/cf.md @@ -255,7 +255,7 @@ extern size_t rank; // Build the CF object and perform the decomposition. // The constructor takes a default-constructed factorizer, which, by default, // is of type NMFALSFactorizer. -CF cf(data, NMFALSFactorizer(), neighborhood, rank); +CFType cf(data, NMFPolicy(), neighborhood, rank); // Store the results in this object. arma::Mat recommendations; @@ -276,6 +276,9 @@ alternating least squares update rules). These include: - `NMFALSFactorizer` - `RegularizedSVD` - `QUIC_SVD` + - `BiasSVD` + - `SVDPlusPlus` + - `RandomizedSVD` The `AMF` class has many other possibilities than those listed here; it is a framework for alternating matrix factorization techniques. See the `AMF` class @@ -297,7 +300,7 @@ extern size_t neighborhood; extern size_t rank; // Build the CF object and perform the decomposition. -CF cf(data, RegularizedSVD(), neighborhood, rank); +CFType cf(data, RegSVDPolicy(), neighborhood, rank); // Store the results in this object. arma::Mat recommendations; @@ -330,7 +333,7 @@ extern size_t rank; // Build the CF object and perform the decomposition. // The constructor takes a default-constructed factorizer, which, by default, // is of type NMFALSFactorizer. -CF cf(data, NMFALSFactorizer(), neighborhood, rank); +CFType cf(data, NMFPolicy(), neighborhood, rank); const double prediction = cf.Predict(12, 50); // User 12, item 50. ``` @@ -356,7 +359,7 @@ extern size_t rank; // Build the CF object and perform the decomposition. // The constructor takes a default-constructed factorizer, which, by default, // is of type NMFALSFactorizer. -CF cf(data, NMFALSFactorizer(), neighborhood, rank); +CFType cf(data, NMFPolicy(), neighborhood, rank); // References to W and H matrices. const arma::mat& W = cf.W(); From 0b9b1d483f1c0caf8c01f4ed0929933391e924e6 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Mon, 6 Feb 2023 10:52:47 +0530 Subject: [PATCH 02/88] Update codes in datasetmapper.md --- doc/tutorials/datasetmapper.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/tutorials/datasetmapper.md b/doc/tutorials/datasetmapper.md index a841071216..a94854568a 100644 --- a/doc/tutorials/datasetmapper.md +++ b/doc/tutorials/datasetmapper.md @@ -43,7 +43,7 @@ function. using namespace mlpack; arma::mat data; -data::DatasetMapper info; +data::DatasetInfo info; data::Load("dataset.csv", data, info); ``` @@ -155,8 +155,8 @@ std::cout << info.UnmapString(1, 2) << "\n"; This will print: ``` -T -F +True +False ``` ### `UnmapValue()` @@ -168,8 +168,8 @@ The `UnmapValue()` function has the signature `UnmapValue(const std::string - `dimension` is the dimension in which you want to find the mapped value ```c++ -std::cout << info.UnmapValue("T", 2) << "\n"; -std::cout << info.UnmapValue("F", 2) << "\n"; +std::cout << info.UnmapValue("True", 2) << "\n"; +std::cout << info.UnmapValue("False", 2) << "\n"; ``` will produce: From 2f3916c8d7c5e4d18ef89e3c8f490c4197859ccb Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Mon, 6 Feb 2023 16:11:37 +0530 Subject: [PATCH 03/88] fixing codes in approx_kfn doc --- doc/tutorials/approx_kfn.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/tutorials/approx_kfn.md b/doc/tutorials/approx_kfn.md index 33a4237d98..ce422f7cbd 100644 --- a/doc/tutorials/approx_kfn.md +++ b/doc/tutorials/approx_kfn.md @@ -812,7 +812,7 @@ extern arma::mat dataset; QDAFN<> qdafn(dataset, 10, 5); // Print the fifth point of the candidate set. -std::cout << ds.CandidateSet(2).col(4).t(); +std::cout << qdafn.CandidateSet(2).col(4).t(); ``` ### Retraining on a new reference set @@ -896,7 +896,7 @@ extern arma::mat querySet; // Construct the object, performing the default dual-tree search with // approximation level epsilon = 0.05. -KFN kfn(dataset, KFN::DUAL_TREE_MODE, 0.05); +KFN kfn(dataset, DUAL_TREE_MODE, 0.05); // Search for approximate furthest neighbors. arma::Mat neighbors; From 30a94a6549c609b3ce7087d959f93284ac05712f Mon Sep 17 00:00:00 2001 From: Aditya Raj <96882869+aadi-raj@users.noreply.github.com> Date: Mon, 6 Feb 2023 17:30:05 +0530 Subject: [PATCH 04/88] Fixed misleading path --- doc/developer/bindings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/developer/bindings.md b/doc/developer/bindings.md index 53f6e11b02..92cb14c889 100644 --- a/doc/developer/bindings.md +++ b/doc/developer/bindings.md @@ -279,7 +279,7 @@ add_all_bindings(program_dir program_name "category") In this example, this will also add a Markdown binding, which will generate documentation that is typically used to build the website. The `category` parameter should be one of the categories in -`src/mlpack/bindings/markdown/MarkdownCategories.cmake`. +`src/mlpack/bindings/Categories.cmake`. ## How to write mlpack bindings From 7b71aa1de9b25c7600da46dc49871305f30bc08a Mon Sep 17 00:00:00 2001 From: Aditya Raj <96882869+aadi-raj@users.noreply.github.com> Date: Mon, 6 Feb 2023 17:41:34 +0530 Subject: [PATCH 05/88] Removed redundant documentation --- src/mlpack/core/util/param.hpp | 312 --------------------------------- 1 file changed, 312 deletions(-) diff --git a/src/mlpack/core/util/param.hpp b/src/mlpack/core/util/param.hpp index 5879476b59..d6faad82a4 100644 --- a/src/mlpack/core/util/param.hpp +++ b/src/mlpack/core/util/param.hpp @@ -69,12 +69,6 @@ * should be present in your program! Therefore, use it in the main.cpp * (or corresponding binding) in your program. * - * @see mlpack::IO, PARAM_FLAG(), PARAM_INT_IN(), PARAM_DOUBLE_IN(), - * PARAM_STRING_IN(), PARAM_VECTOR_IN(), PARAM_INT_OUT(), PARAM_DOUBLE_OUT(), - * PARAM_VECTOR_OUT(), PARAM_INT_IN_REQ(), PARAM_DOUBLE_IN_REQ(), - * PARAM_STRING_IN_REQ(), PARAM_VECTOR_IN_REQ(), PARAM_INT_OUT_REQ(), - * PARAM_DOUBLE_OUT_REQ(), PARAM_VECTOR_OUT_REQ(), PARAM_STRING_OUT_REQ(). - * * @param SHORT_DESC Short two-sentence description of the program; it should * describe what the program implements and does, and a quick overview of * how it can be used and what it should be used for. @@ -100,12 +94,6 @@ * If you wish to "revamp" some bindings, then use the BINDING_LONG_DESC() * of the method that you pass first into the group_bindings() macro. For all other * methods, it is fine if you keep the BINDING_LONG_DESC() empty. - * - * @see mlpack::IO, PARAM_FLAG(), PARAM_INT_IN(), PARAM_DOUBLE_IN(), - * PARAM_STRING_IN(), PARAM_VECTOR_IN(), PARAM_INT_OUT(), PARAM_DOUBLE_OUT(), - * PARAM_VECTOR_OUT(), PARAM_INT_IN_REQ(), PARAM_DOUBLE_IN_REQ(), - * PARAM_STRING_IN_REQ(), PARAM_VECTOR_IN_REQ(), PARAM_INT_OUT_REQ(), - * PARAM_DOUBLE_OUT_REQ(), PARAM_VECTOR_OUT_REQ(), PARAM_STRING_OUT_REQ(). * * @param LONG_DESC Long string describing what the program does. Newlines * should not be used here; this is taken care of by IO (however, you @@ -132,12 +120,6 @@ * present in your program! Therefore, use it in the main.cpp * (or corresponding binding) in your program. * - * @see mlpack::IO, PARAM_FLAG(), PARAM_INT_IN(), PARAM_DOUBLE_IN(), - * PARAM_STRING_IN(), PARAM_VECTOR_IN(), PARAM_INT_OUT(), PARAM_DOUBLE_OUT(), - * PARAM_VECTOR_OUT(), PARAM_INT_IN_REQ(), PARAM_DOUBLE_IN_REQ(), - * PARAM_STRING_IN_REQ(), PARAM_VECTOR_IN_REQ(), PARAM_INT_OUT_REQ(), - * PARAM_DOUBLE_OUT_REQ(), PARAM_VECTOR_OUT_REQ(), PARAM_STRING_OUT_REQ(). - * * @param EXAMPLE Long string describing a simple usage example.. Newlines * should not be used here; this is taken care of by IO (however, you * can explicitly specify newlines to denote new paragraphs). You can @@ -163,12 +145,6 @@ * present in your program! Therefore, use it in the main.cpp * (or corresponding binding) in your program. * - * @see mlpack::IO, PARAM_FLAG(), PARAM_INT_IN(), PARAM_DOUBLE_IN(), - * PARAM_STRING_IN(), PARAM_VECTOR_IN(), PARAM_INT_OUT(), PARAM_DOUBLE_OUT(), - * PARAM_VECTOR_OUT(), PARAM_INT_IN_REQ(), PARAM_DOUBLE_IN_REQ(), - * PARAM_STRING_IN_REQ(), PARAM_VECTOR_IN_REQ(), PARAM_INT_OUT_REQ(), - * PARAM_DOUBLE_OUT_REQ(), PARAM_VECTOR_OUT_REQ(), PARAM_STRING_OUT_REQ(). - * * Provide a link for a binding's "see also" documentation section, which is * primarily (but not necessarily exclusively) used by the Markdown bindings * This link can be specified by calling SEE_ALSO("description", "link"), where @@ -230,18 +206,6 @@ * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). * @param DEF Default value of the parameter. - * - * @see mlpack::IO, BINDING_USER_NAME(), BINDING_SHORT_DESC(), - * BINDING_LONG_DESC(), BINDING_EXAMPLE() and BINDING_SEE_ALSO(). - * - * @bug -// Use a forward declaration of the class. - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_INT_IN(ID, DESC, ALIAS, DEF) \ PARAM_IN(int, ID, DESC, ALIAS, DEF, false) @@ -262,17 +226,6 @@ * @param DESC Quick description of the parameter (1-2 sentences). Don't use * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. - * - * @see mlpack::IO, BINDING_USER_NAME(), BINDING_SHORT_DESC(), - * BINDING_LONG_DESC(), BINDING_EXAMPLE() and BINDING_SEE_ALSO(). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_INT_OUT(ID, DESC) \ PARAM_OUT(int, ID, DESC, "", 0, false) @@ -289,17 +242,6 @@ * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). * @param DEF Default value of the parameter. - * - * @see mlpack::IO, BINDING_USER_NAME(), BINDING_SHORT_DESC(), - * BINDING_LONG_DESC(), BINDING_EXAMPLE() and BINDING_SEE_ALSO(). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_DOUBLE_IN(ID, DESC, ALIAS, DEF) \ PARAM_IN(double, ID, DESC, ALIAS, DEF, false) @@ -320,17 +262,6 @@ * @param DESC Quick description of the parameter (1-2 sentences). Don't use * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. - * - * @see mlpack::IO, BINDING_USER_NAME(), BINDING_SHORT_DESC(), - * BINDING_LONG_DESC(), BINDING_EXAMPLE() and BINDING_SEE_ALSO(). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_DOUBLE_OUT(ID, DESC) \ PARAM_OUT(double, ID, DESC, "", 0.0, false) @@ -349,17 +280,6 @@ * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). * @param DEF Default value of the parameter. - * - * @see mlpack::IO, BINDING_USER_NAME(), BINDING_SHORT_DESC(), - * BINDING_LONG_DESC(), BINDING_EXAMPLE() and BINDING_SEE_ALSO(). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_STRING_IN(ID, DESC, ALIAS, DEF) \ PARAM_IN(std::string, ID, DESC, ALIAS, DEF, false) @@ -381,17 +301,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @see mlpack::IO, BINDING_USER_NAME(), BINDING_SHORT_DESC(), - * BINDING_LONG_DESC(), BINDING_EXAMPLE() and BINDING_SEE_ALSO(). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_STRING_OUT(ID, DESC, ALIAS) \ PARAM_OUT(std::string, ID, DESC, ALIAS, "", false) @@ -412,14 +321,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_MATRIX_IN(ID, DESC, ALIAS) \ PARAM_MATRIX(ID, DESC, ALIAS, false, true, true) @@ -440,14 +341,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_MATRIX_IN_REQ(ID, DESC, ALIAS) \ PARAM_MATRIX(ID, DESC, ALIAS, true, true, true) @@ -473,14 +366,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_MATRIX_OUT(ID, DESC, ALIAS) \ PARAM_MATRIX(ID, DESC, ALIAS, false, true, false) @@ -502,14 +387,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_TMATRIX_IN(ID, DESC, ALIAS) \ PARAM_MATRIX(ID, DESC, ALIAS, false, false, true) @@ -532,14 +409,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_TMATRIX_IN_REQ(ID, DESC, ALIAS) \ PARAM_MATRIX(ID, DESC, ALIAS, true, false, true) @@ -567,14 +436,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_TMATRIX_OUT(ID, DESC, ALIAS) \ PARAM_MATRIX(ID, DESC, ALIAS, false, false, false) @@ -595,14 +456,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_UMATRIX_IN(ID, DESC, ALIAS) \ PARAM_UMATRIX(ID, DESC, ALIAS, false, true, true) @@ -624,14 +477,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_UMATRIX_IN_REQ(ID, DESC, ALIAS) \ PARAM_UMATRIX(ID, DESC, ALIAS, true, true, true) @@ -658,14 +503,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_UMATRIX_OUT(ID, DESC, ALIAS) \ PARAM_UMATRIX(ID, DESC, ALIAS, false, true, false) @@ -687,14 +524,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_COL_IN(ID, DESC, ALIAS) \ PARAM_COL(ID, DESC, ALIAS, false, true, true) @@ -715,14 +544,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_COL_IN_REQ(ID, DESC, ALIAS) \ PARAM_COL(ID, DESC, ALIAS, true, true, true) @@ -743,14 +564,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_ROW_IN(ID, DESC, ALIAS) \ PARAM_ROW(ID, DESC, ALIAS, false, true, true) @@ -771,14 +584,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_UCOL_IN(ID, DESC, ALIAS) \ PARAM_UCOL(ID, DESC, ALIAS, false, true, true) @@ -800,14 +605,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_UROW_IN(ID, DESC, ALIAS) \ PARAM_UROW(ID, DESC, ALIAS, false, true, true) @@ -833,14 +630,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_COL_OUT(ID, DESC, ALIAS) \ PARAM_COL(ID, DESC, ALIAS, false, true, false) @@ -866,14 +655,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_ROW_OUT(ID, DESC, ALIAS) \ PARAM_ROW(ID, DESC, ALIAS, false, true, false) @@ -899,14 +680,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_UCOL_OUT(ID, DESC, ALIAS) \ PARAM_UCOL(ID, DESC, ALIAS, false, true, false) @@ -932,14 +705,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_UROW_OUT(ID, DESC, ALIAS) \ PARAM_UROW(ID, DESC, ALIAS, false, true, false) @@ -956,17 +721,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @see mlpack::IO, BINDING_USER_NAME(), BINDING_SHORT_DESC(), - * BINDING_LONG_DESC(), BINDING_EXAMPLE() and BINDING_SEE_ALSO(). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_VECTOR_IN(T, ID, DESC, ALIAS) \ PARAM_IN(std::vector, ID, DESC, ALIAS, std::vector(), false) @@ -990,17 +744,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @see mlpack::IO, BINDING_USER_NAME(), BINDING_SHORT_DESC(), - * BINDING_LONG_DESC(), BINDING_EXAMPLE() and BINDING_SEE_ALSO(). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_VECTOR_OUT(T, ID, DESC, ALIAS) \ PARAM_OUT(std::vector, ID, DESC, ALIAS, std::vector(), false) @@ -1030,17 +773,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS One-character string representing the alias of the parameter. - * - * @see mlpack::IO, BINDING_USER_NAME(), BINDING_SHORT_DESC(), - * BINDING_LONG_DESC(), BINDING_EXAMPLE() and BINDING_SEE_ALSO(). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define TUPLE_TYPE std::tuple #define PARAM_MATRIX_AND_INFO_IN(ID, DESC, ALIAS) \ @@ -1145,17 +877,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @see mlpack::IO, BINDING_USER_NAME(), BINDING_SHORT_DESC(), - * BINDING_LONG_DESC(), BINDING_EXAMPLE() and BINDING_SEE_ALSO(). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_INT_IN_REQ(ID, DESC, ALIAS) \ PARAM_IN(int, ID, DESC, ALIAS, 0, true) @@ -1170,17 +891,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @see mlpack::IO, BINDING_USER_NAME(), BINDING_SHORT_DESC(), - * BINDING_LONG_DESC(), BINDING_EXAMPLE() and BINDING_SEE_ALSO(). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_DOUBLE_IN_REQ(ID, DESC, ALIAS) \ PARAM_IN(double, ID, DESC, ALIAS, 0.0, true) @@ -1195,17 +905,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @see mlpack::IO, BINDING_USER_NAME(), BINDING_SHORT_DESC(), - * BINDING_LONG_DESC(), BINDING_EXAMPLE() and BINDING_SEE_ALSO(). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_STRING_IN_REQ(ID, DESC, ALIAS) \ PARAM_IN(std::string, ID, DESC, ALIAS, "", true) @@ -1222,17 +921,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @see mlpack::IO, BINDING_USER_NAME(), BINDING_SHORT_DESC(), - * BINDING_LONG_DESC(), BINDING_EXAMPLE() and BINDING_SEE_ALSO(). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_VECTOR_IN_REQ(T, ID, DESC, ALIAS) \ PARAM_IN(std::vector, ID, DESC, ALIAS, std::vector(), true); From f1ed74ee94445b445814b6b62db1f37cfc1ffd12 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Mon, 6 Feb 2023 21:15:31 +0530 Subject: [PATCH 06/88] improve doc --- doc/tutorials/fastmks.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/tutorials/fastmks.md b/doc/tutorials/fastmks.md index e25fe766fd..21a0d4afb8 100644 --- a/doc/tutorials/fastmks.md +++ b/doc/tutorials/fastmks.md @@ -257,6 +257,9 @@ manually specified. Choices that mlpack provides include: - `HyperbolicTangentKernel` - `LaplacianKernel` - `PSpectrumStringKernel` + - `CauchyKernal` + - `ExampleKernal` + - `SphericalKernal` The following examples use kernels from that list. Writing your own kernel is detailed in the next section. Remember that when you are using the C++ @@ -293,7 +296,7 @@ f.Search(5, indices, products); In this setting we have both a query and reference dataset. We search for 10 maximum kernels. -``` +```c++ #include using namespace mlpack::fastmks; From 2a3f3e69d1051d47743e8972f27e22903a5a40de Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Tue, 7 Feb 2023 19:12:27 +0530 Subject: [PATCH 07/88] CFType to CF Change inorder to make CF typedef from CFType --- doc/tutorials/cf.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/tutorials/cf.md b/doc/tutorials/cf.md index 2efb086e9d..397f5708da 100644 --- a/doc/tutorials/cf.md +++ b/doc/tutorials/cf.md @@ -255,7 +255,7 @@ extern size_t rank; // Build the CF object and perform the decomposition. // The constructor takes a default-constructed factorizer, which, by default, // is of type NMFALSFactorizer. -CFType cf(data, NMFPolicy(), neighborhood, rank); +CF cf(data, NMFPolicy(), neighborhood, rank); // Store the results in this object. arma::Mat recommendations; @@ -300,7 +300,7 @@ extern size_t neighborhood; extern size_t rank; // Build the CF object and perform the decomposition. -CFType cf(data, RegSVDPolicy(), neighborhood, rank); +CF cf(data, RegSVDPolicy(), neighborhood, rank); // Store the results in this object. arma::Mat recommendations; @@ -333,7 +333,7 @@ extern size_t rank; // Build the CF object and perform the decomposition. // The constructor takes a default-constructed factorizer, which, by default, // is of type NMFALSFactorizer. -CFType cf(data, NMFPolicy(), neighborhood, rank); +CF cf(data, NMFPolicy(), neighborhood, rank); const double prediction = cf.Predict(12, 50); // User 12, item 50. ``` @@ -359,7 +359,7 @@ extern size_t rank; // Build the CF object and perform the decomposition. // The constructor takes a default-constructed factorizer, which, by default, // is of type NMFALSFactorizer. -CFType cf(data, NMFPolicy(), neighborhood, rank); +CF cf(data, NMFPolicy(), neighborhood, rank); // References to W and H matrices. const arma::mat& W = cf.W(); From 50c797a9b1151f025b1850ef54c29ea47ce75005 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Tue, 7 Feb 2023 19:29:32 +0530 Subject: [PATCH 08/88] typedef CF to CFType --- src/mlpack/methods/cf/cf.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mlpack/methods/cf/cf.hpp b/src/mlpack/methods/cf/cf.hpp index 90bcb80338..6efb558d12 100644 --- a/src/mlpack/methods/cf/cf.hpp +++ b/src/mlpack/methods/cf/cf.hpp @@ -283,6 +283,8 @@ class CFType }; }; // class CFType +typedef CFType<> CF; + } // namespace mlpack // Include implementation of templated functions. From b99e7cf9b8ee91c1eea75580c1c0b7a1c5e4b59b Mon Sep 17 00:00:00 2001 From: AdarshSantoria Date: Thu, 9 Feb 2023 09:22:20 +0530 Subject: [PATCH 09/88] add quic_svd for cf --- .../decomposition_policies.hpp | 1 + .../quic_svd_method.hpp | 174 ++++++++++++++++++ src/mlpack/methods/quic_svd/quic_svd.hpp | 51 +++-- src/mlpack/methods/quic_svd/quic_svd_impl.hpp | 26 ++- 4 files changed, 237 insertions(+), 15 deletions(-) create mode 100644 src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp diff --git a/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp b/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp index 731b8db8d1..82b1129e01 100644 --- a/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp +++ b/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp @@ -20,5 +20,6 @@ #include "svd_complete_method.hpp" #include "svd_incomplete_method.hpp" #include "svdplusplus_method.hpp" +#include "quic_svd_method.hpp" #endif diff --git a/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp b/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp new file mode 100644 index 0000000000..297d560217 --- /dev/null +++ b/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp @@ -0,0 +1,174 @@ +/** + * @file methods/cf/decomposition_policies/quic_svd_method.hpp + * @author Adarsh Santoria + * + * Implementation of the quic svd method for use in + * Collaborative Fitlering. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_METHODS_CF_DECOMPOSITION_POLICIES_QUIC_SVD_METHOD_HPP +#define MLPACK_METHODS_CF_DECOMPOSITION_POLICIES_QUIC_SVD_METHOD_HPP + +#include +#include + +namespace mlpack { + +/** + * Implementation of the Quic SVD policy to act as a wrapper when + * accessing Quic SVD from within CFType. + * + * An example of how to use QuicSVDPolicy in CF is shown below: + * + * @code + * extern arma::mat data; // data is a (user, item, rating) table. + * // Users for whom recommendations are generated. + * extern arma::Col users; + * arma::Mat recommendations; // Resulting recommendations. + * + * CFType cf(data); + * + * // Generate 10 recommendations for all users. + * cf.GetRecommendations(10, recommendations); + * @endcode + */ +class QuicSVDPolicy +{ + public: + /** + * Use quic SVD method to perform collaborative filtering + */ + QuicSVDPolicy() + { + /* Nothing to do here */ + } + + /** + * Apply Collaborative Filtering to the provided data set using the + * quic SVD. + * + * @param * (data) Data matrix: dense matrix (coordinate lists) + * or sparse matrix(cleaned). + * @param cleanedData item user table in form of sparse matrix. + * @param * (rank) Rank parameter for matrix factorization. + * @param * (maxIterations) Maximum number of iterations. + * @param * (minResidue) Residue required to terminate. + * @param * (mit) Whether to terminate only when maxIterations is reached. + */ + template + void Apply(const MatType& data, + const arma::sp_mat& /* cleanedData */, + const size_t /* rank */, + const size_t /* maxIterations */, + const double /* minResidue */, + const bool /* mit */) + { + arma::mat sigma; + + // Do singular value decomposition using the quic SVD algorithm. + QUIC_SVD quicsvd; + quicsvd.Apply(data, w, h, sigma); + + if (data.n_cols < data.n_rows) + { + arma::mat tempMat = w; + w = h; + h = tempMat; + } + + // Take transpose of the matrix h as required by CF class. + h = arma::trans(h); + } + + /** + * Return predicted rating given user ID and item ID. + * + * @param user User ID. + * @param item Item ID. + */ + double GetRating(const size_t user, const size_t item) const + { + double rating = arma::as_scalar(w.row(item) * h.col(user)); + return rating; + } + + /** + * Get predicted ratings for a user. + * + * @param user User ID. + * @param rating Resulting rating vector. + */ + void GetRatingOfUser(const size_t user, arma::vec& rating) const + { + rating = w * h.col(user); + } + + /** + * Get the neighborhood and corresponding similarities for a set of users. + * + * @tparam NeighborSearchPolicy The policy to perform neighbor search. + * + * @param users Users whose neighborhood is to be computed. + * @param numUsersForSimilarity The number of neighbors returned for + * each user. + * @param neighborhood Neighbors represented by user IDs. + * @param similarities Similarity between each user and each of its + * neighbors. + */ + template + void GetNeighborhood(const arma::Col& users, + const size_t numUsersForSimilarity, + arma::Mat& neighborhood, + arma::mat& similarities) const + { + // We want to avoid calculating the full rating matrix, so we will do + // nearest neighbor search only on the H matrix, using the observation that + // if the rating matrix X = W*H, then d(X.col(i), X.col(j)) = d(W H.col(i), + // W H.col(j)). This can be seen as nearest neighbor search on the H + // matrix with the Mahalanobis distance where M^{-1} = W^T W. So, we'll + // decompose M^{-1} = L L^T (the Cholesky decomposition), and then multiply + // H by L^T. Then we can perform nearest neighbor search. + arma::mat l = arma::chol(w.t() * w); + arma::mat stretchedH = l * h; // Due to the Armadillo API, l is L^T. + + // Temporarily store feature vector of queried users. + arma::mat query(stretchedH.n_rows, users.n_elem); + // Select feature vectors of queried users. + for (size_t i = 0; i < users.n_elem; ++i) + query.col(i) = stretchedH.col(users(i)); + + NeighborSearchPolicy neighborSearch(stretchedH); + neighborSearch.Search( + query, numUsersForSimilarity, neighborhood, similarities); + } + + //! Get the Item Matrix. + const arma::mat& W() const { return w; } + //! Get the User Matrix. + const arma::mat& H() const { return h; } + + /** + * Serialization. + */ + template + void serialize(Archive& ar, const uint32_t /* version */) + { + ar(CEREAL_NVP(w)); + ar(CEREAL_NVP(h)); + } + + private: + //! Item matrix. + arma::mat w; + //! User matrix. + arma::mat h; +}; + +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/quic_svd/quic_svd.hpp b/src/mlpack/methods/quic_svd/quic_svd.hpp index bdf9d0d24a..360fbf1763 100644 --- a/src/mlpack/methods/quic_svd/quic_svd.hpp +++ b/src/mlpack/methods/quic_svd/quic_svd.hpp @@ -42,21 +42,20 @@ namespace mlpack { * const double epsilon = 0.01; // Relative error limit of data in subspace. * const double delta = 0.1 // Lower error bound for Monte Carlo estimate. * + * // Make a QuicSVD object. + * QuicSVD qSVD(); + * * arma::mat u, v, sigma; // Matrices for the factors. data = u * sigma * v.t() - * - * // Get the factorization in the constructor. - * QUIC_SVD(data, u, v, sigma, epsilon, delta); + * + * // Use the Apply() method to get a factorization. + * qSVD.Apply(data, u, v, sigma, epsilon, delta); * @endcode */ class QUIC_SVD { public: /** - * Constructor which implements the QUIC-SVD algorithm. The function calls the - * CosineTree constructor to create a subspace basis, where the original - * matrix's projection has minimum reconstruction error. The constructor then - * uses the ExtractSVD() function to calculate the SVD of the original dataset - * in that subspace. + * Create object for the randomized SVD method. * * @param dataset Matrix for which SVD is calculated. * @param u First unitary matrix. @@ -72,6 +71,35 @@ class QUIC_SVD const double epsilon = 0.03, const double delta = 0.1); + /** + * Create object for the randomized SVD method. + * + * @param epsilon Error tolerance fraction for calculated subspace. + * @param delta Cumulative probability for Monte Carlo error lower bound. + */ + QUIC_SVD(const double epsilon = 0.03, + const double delta = 0.1); + + /** + * The function calls the CosineTree constructor to create a subspace basis, + * where the original matrix's projection has minimum reconstruction error. + * The constructor then uses the ExtractSVD() function to calculate the SVD + * of the original dataset in that subspace. + * + * @param dataset Matrix for which SVD is calculated. + * @param u First unitary matrix. + * @param v Second unitary matrix. + * @param sigma Diagonal matrix of singular values. + * @param epsilon Error tolerance fraction for calculated subspace. + * @param delta Cumulative probability for Monte Carlo error lower bound. + */ + void Apply(const arma::mat& dataset, + arma::mat& u, + arma::mat& v, + arma::mat& sigma, + const double epsilon = 0.03, + const double delta = 0.1); + /** * This function uses the vector subspace created using a cosine tree to * calculate an approximate SVD of the original matrix. @@ -80,11 +108,12 @@ class QUIC_SVD * @param v Second unitary matrix. * @param sigma Diagonal matrix of singular values. */ - void ExtractSVD(arma::mat& u, arma::mat& v, arma::mat& sigma); + void ExtractSVD(const arma::mat& dataset, + arma::mat& u, + arma::mat& v, + arma::mat& sigma); private: - //! Matrix for which cosine tree is constructed. - const arma::mat& dataset; //! Subspace basis of the input dataset. arma::mat basis; }; diff --git a/src/mlpack/methods/quic_svd/quic_svd_impl.hpp b/src/mlpack/methods/quic_svd/quic_svd_impl.hpp index 9d009c9751..3bbc551f9f 100644 --- a/src/mlpack/methods/quic_svd/quic_svd_impl.hpp +++ b/src/mlpack/methods/quic_svd/quic_svd_impl.hpp @@ -23,8 +23,25 @@ inline QUIC_SVD::QUIC_SVD( arma::mat& v, arma::mat& sigma, const double epsilon, - const double delta) : - dataset(dataset) + const double delta) +{ + Apply(dataset, u, v, sigma, epsilon, delta); +} + +inline QUIC_SVD::QUIC_SVD( + const double epsilon, + const double delta) +{ + /* Nothing to do here */ +} + +inline void QUIC_SVD::Apply( + const arma::mat& dataset, + arma::mat& u, + arma::mat& v, + arma::mat& sigma, + const double epsilon, + const double delta) { // Since columns are sample in the implementation, the matrix is transposed if // necessary for maximum speedup. @@ -42,10 +59,11 @@ inline QUIC_SVD::QUIC_SVD( // Use the ExtractSVD algorithm mentioned in the paper to extract the SVD of // the original dataset in the obtained subspace. - ExtractSVD(u, v, sigma); + ExtractSVD(dataset,u, v, sigma); } -inline void QUIC_SVD::ExtractSVD(arma::mat& u, +inline void QUIC_SVD::ExtractSVD(const arma::mat& dataset, + arma::mat& u, arma::mat& v, arma::mat& sigma) { From a57b5617c9d62e9c98bb6f58d6052c373f36d313 Mon Sep 17 00:00:00 2001 From: AdarshSantoria Date: Thu, 9 Feb 2023 09:42:16 +0530 Subject: [PATCH 10/88] fixing comments --- .../methods/cf/decomposition_policies/quic_svd_method.hpp | 4 ++-- src/mlpack/methods/quic_svd/quic_svd.hpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp b/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp index 297d560217..f6eb4cb272 100644 --- a/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp +++ b/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp @@ -52,9 +52,9 @@ class QuicSVDPolicy * Apply Collaborative Filtering to the provided data set using the * quic SVD. * - * @param * (data) Data matrix: dense matrix (coordinate lists) + * @param data Data matrix: dense matrix (coordinate lists) * or sparse matrix(cleaned). - * @param cleanedData item user table in form of sparse matrix. + * @param * (cleanedData) item user table in form of sparse matrix. * @param * (rank) Rank parameter for matrix factorization. * @param * (maxIterations) Maximum number of iterations. * @param * (minResidue) Residue required to terminate. diff --git a/src/mlpack/methods/quic_svd/quic_svd.hpp b/src/mlpack/methods/quic_svd/quic_svd.hpp index 360fbf1763..861494f2d1 100644 --- a/src/mlpack/methods/quic_svd/quic_svd.hpp +++ b/src/mlpack/methods/quic_svd/quic_svd.hpp @@ -46,7 +46,7 @@ namespace mlpack { * QuicSVD qSVD(); * * arma::mat u, v, sigma; // Matrices for the factors. data = u * sigma * v.t() - * + * * // Use the Apply() method to get a factorization. * qSVD.Apply(data, u, v, sigma, epsilon, delta); * @endcode From 58df0e6908cef80931dca3a3aa4a20afaa39cd05 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Thu, 9 Feb 2023 14:05:11 +0530 Subject: [PATCH 11/88] default set only for NMFPolicy --- doc/tutorials/cf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/cf.md b/doc/tutorials/cf.md index 397f5708da..8a932d0df5 100644 --- a/doc/tutorials/cf.md +++ b/doc/tutorials/cf.md @@ -300,7 +300,7 @@ extern size_t neighborhood; extern size_t rank; // Build the CF object and perform the decomposition. -CF cf(data, RegSVDPolicy(), neighborhood, rank); +CFType cf(data, RegSVDPolicy(), neighborhood, rank); // Store the results in this object. arma::Mat recommendations; From 0efb5acde3643f65d8e28ca5a7e985938f2d33fc Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Fri, 10 Feb 2023 00:53:46 +0530 Subject: [PATCH 12/88] Update src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp Co-authored-by: Ryan Curtin --- .../methods/cf/decomposition_policies/quic_svd_method.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp b/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp index f6eb4cb272..d2cfefb76d 100644 --- a/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp +++ b/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp @@ -20,7 +20,7 @@ namespace mlpack { /** - * Implementation of the Quic SVD policy to act as a wrapper when + * Implementation of the QUIC-SVD policy to act as a wrapper when * accessing Quic SVD from within CFType. * * An example of how to use QuicSVDPolicy in CF is shown below: From c223e6464030c5cb09acc670057ed7e9558391a0 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Fri, 10 Feb 2023 00:53:56 +0530 Subject: [PATCH 13/88] Update src/mlpack/methods/quic_svd/quic_svd.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/quic_svd/quic_svd.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/quic_svd/quic_svd.hpp b/src/mlpack/methods/quic_svd/quic_svd.hpp index 861494f2d1..0e0e40896e 100644 --- a/src/mlpack/methods/quic_svd/quic_svd.hpp +++ b/src/mlpack/methods/quic_svd/quic_svd.hpp @@ -72,7 +72,7 @@ class QUIC_SVD const double delta = 0.1); /** - * Create object for the randomized SVD method. + * Create object for the QUIC-SVD method. * * @param epsilon Error tolerance fraction for calculated subspace. * @param delta Cumulative probability for Monte Carlo error lower bound. From 4d8f0000832d312468e2770e8315327f5826e8e0 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Fri, 10 Feb 2023 01:07:06 +0530 Subject: [PATCH 14/88] fix style --- .../cf/decomposition_policies/quic_svd_method.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp b/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp index d2cfefb76d..e199359f2d 100644 --- a/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp +++ b/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp @@ -74,12 +74,12 @@ class QuicSVDPolicy QUIC_SVD quicsvd; quicsvd.Apply(data, w, h, sigma); - if (data.n_cols < data.n_rows) - { - arma::mat tempMat = w; - w = h; - h = tempMat; - } + if (data.n_cols < data.n_rows) + { + arma::mat tempMat = w; + w = h; + h = tempMat; + } // Take transpose of the matrix h as required by CF class. h = arma::trans(h); From 7a6c51615b2c6ed041ddc0a48e5e27eca4d75c99 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Fri, 10 Feb 2023 01:30:52 +0530 Subject: [PATCH 15/88] Change factorizers names to what's used in code --- doc/tutorials/cf.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/tutorials/cf.md b/doc/tutorials/cf.md index 8a932d0df5..8cb821c7cd 100644 --- a/doc/tutorials/cf.md +++ b/doc/tutorials/cf.md @@ -270,15 +270,15 @@ mlpack provides a number of existing factorizers which can be used in place of the default `NMFALSFactorizer` (which is non-negative matrix factorization with alternating least squares update rules). These include: - - `SVDBatchFactorizer` - - `SVDCompleteIncrementalFactorizer` - - `SVDIncompleteIncrementalFactorizer` - - `NMFALSFactorizer` - - `RegularizedSVD` - - `QUIC_SVD` - - `BiasSVD` - - `SVDPlusPlus` - - `RandomizedSVD` + - `BatchSVDPolicy` + - `SVDCompletePolicy` + - `SVDIncompletePolicy` + - `NMFPolicy` + - `RegSVDPolicy` + - `QUIC_SVDPolicy` + - `BiasSVDPolicy` + - `SVDPlusPlusPolicy` + - `RandomizedSVDPolicy` The `AMF` class has many other possibilities than those listed here; it is a framework for alternating matrix factorization techniques. See the `AMF` class From c26e762a57ce300079ad548eba0872406292fd8d Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Fri, 10 Feb 2023 01:33:08 +0530 Subject: [PATCH 16/88] Change factorizers names to it's corresponding --- doc/tutorials/cf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/cf.md b/doc/tutorials/cf.md index 8cb821c7cd..aa3c4fb57e 100644 --- a/doc/tutorials/cf.md +++ b/doc/tutorials/cf.md @@ -275,7 +275,7 @@ alternating least squares update rules). These include: - `SVDIncompletePolicy` - `NMFPolicy` - `RegSVDPolicy` - - `QUIC_SVDPolicy` + - `QuicSVDPolicy` - `BiasSVDPolicy` - `SVDPlusPlusPolicy` - `RandomizedSVDPolicy` From 58026f687b3c27ff644b209480326006110f4767 Mon Sep 17 00:00:00 2001 From: conradsnicta Date: Fri, 10 Feb 2023 12:42:08 +0100 Subject: [PATCH 17/88] avoid deprecation warnings in Armadillo 11.4.4+ (#3405) --- HISTORY.md | 2 ++ src/mlpack/core/data/save_impl.hpp | 14 +++++++------- src/mlpack/tests/load_save_test.cpp | 18 +++++++++--------- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 36232c4eeb..a3e321e1e3 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -6,6 +6,8 @@ * Fix DBSCAN handling of non-core points (#3346). + * Avoid deprecation warnings in Armadillo 11.4.4+ (#3405). + ### mlpack 4.0.1 ###### 2022-12-23 * Fix mapping of categorical data for Julia bindings (#3305). diff --git a/src/mlpack/core/data/save_impl.hpp b/src/mlpack/core/data/save_impl.hpp index 2d3c92e9ff..821a1c4ca3 100644 --- a/src/mlpack/core/data/save_impl.hpp +++ b/src/mlpack/core/data/save_impl.hpp @@ -101,10 +101,10 @@ bool Save(const std::string& filename, #ifdef ARMA_USE_HDF5 // We can't save with streams for HDF5. const bool success = (saveType == FileType::HDF5Binary) ? - tmp.quiet_save(filename, ToArmaFileType(saveType)) : - tmp.quiet_save(stream, ToArmaFileType(saveType)); + tmp.save(filename, ToArmaFileType(saveType)) : + tmp.save(stream, ToArmaFileType(saveType)); #else - const bool success = tmp.quiet_save(stream, ToArmaFileType(saveType)); + const bool success = tmp.save(stream, ToArmaFileType(saveType)); #endif if (!success) { @@ -122,10 +122,10 @@ bool Save(const std::string& filename, #ifdef ARMA_USE_HDF5 // We can't save with streams for HDF5. const bool success = (saveType == FileType::HDF5Binary) ? - matrix.quiet_save(filename, ToArmaFileType(saveType)) : - matrix.quiet_save(stream, ToArmaFileType(saveType)); + matrix.save(filename, ToArmaFileType(saveType)) : + matrix.save(stream, ToArmaFileType(saveType)); #else - const bool success = matrix.quiet_save(stream, ToArmaFileType(saveType)); + const bool success = matrix.save(stream, ToArmaFileType(saveType)); #endif if (!success) { @@ -236,7 +236,7 @@ bool Save(const std::string& filename, tmp = trans(matrix); } - const bool success = tmp.quiet_save(stream, ToArmaFileType(saveType)); + const bool success = tmp.save(stream, ToArmaFileType(saveType)); if (!success) { Timer::Stop("saving_data"); diff --git a/src/mlpack/tests/load_save_test.cpp b/src/mlpack/tests/load_save_test.cpp index a4dcfd03ce..6329481d43 100644 --- a/src/mlpack/tests/load_save_test.cpp +++ b/src/mlpack/tests/load_save_test.cpp @@ -58,7 +58,7 @@ TEST_CASE("WrongExtensionWrongLoad", "[LoadSaveTest]") "4 8;"; arma::mat testTrans = trans(test); - REQUIRE(testTrans.quiet_save("test_file.csv", arma::arma_binary) == true); + REQUIRE(testTrans.save("test_file.csv", arma::arma_binary) == true); // Now reload through our interface. REQUIRE(data::Load("test_file.csv", test) == false); @@ -79,7 +79,7 @@ TEST_CASE("WrongExtensionCorrectLoad", "[LoadSaveTest]") "4 8;"; arma::mat testTrans = trans(test); - REQUIRE(testTrans.quiet_save("test_file.csv", arma::arma_binary) == true); + REQUIRE(testTrans.save("test_file.csv", arma::arma_binary) == true); // Now reload through our interface. REQUIRE( @@ -924,7 +924,7 @@ TEST_CASE("LoadArmaBinaryTest", "[LoadSaveTest]") "4 8;"; arma::mat testTrans = trans(test); - REQUIRE(testTrans.quiet_save("test_file.bin", arma::arma_binary) + REQUIRE(testTrans.save("test_file.bin", arma::arma_binary) == true); // Now reload through our interface. @@ -1001,7 +1001,7 @@ TEST_CASE("LoadRawBinaryTest", "[LoadSaveTest]") "7 8;"; arma::mat testTrans = trans(test); - REQUIRE(testTrans.quiet_save("test_file.bin", arma::raw_binary) + REQUIRE(testTrans.save("test_file.bin", arma::raw_binary) == true); // Now reload through our interface. @@ -1028,7 +1028,7 @@ TEST_CASE("LoadPGMBinaryTest", "[LoadSaveTest]") "4 8;"; arma::mat testTrans = trans(test); - REQUIRE(testTrans.quiet_save("test_file.pgm", arma::pgm_binary) + REQUIRE(testTrans.save("test_file.pgm", arma::pgm_binary) == true); // Now reload through our interface. @@ -1080,13 +1080,13 @@ TEST_CASE("LoadHDF5Test", "[LoadSaveTest]") "3 7;" "4 8;"; arma::mat testTrans = trans(test); - REQUIRE(testTrans.quiet_save("test_file.h5", arma::hdf5_binary) + REQUIRE(testTrans.save("test_file.h5", arma::hdf5_binary) == true); - REQUIRE(testTrans.quiet_save("test_file.hdf5", arma::hdf5_binary) + REQUIRE(testTrans.save("test_file.hdf5", arma::hdf5_binary) == true); - REQUIRE(testTrans.quiet_save("test_file.hdf", arma::hdf5_binary) + REQUIRE(testTrans.save("test_file.hdf", arma::hdf5_binary) == true); - REQUIRE(testTrans.quiet_save("test_file.he5", arma::hdf5_binary) + REQUIRE(testTrans.save("test_file.he5", arma::hdf5_binary) == true); // Now reload through our interface. From fcab7355f6d1e154784f89e2a62595a97b69635e Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Fri, 10 Feb 2023 21:09:27 +0530 Subject: [PATCH 18/88] Update HISTORY.md --- HISTORY.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 5abe54ee22..f7d8b9fd17 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,8 @@ ### mlpack ?.?.? ###### ????-??-?? + + * Update outdated codes in cf.md and datasetmapper.md (#3398). + * Bugfix for non-square convolution kernels (#3376). * Fix a few missing includes in `` (#3374). From e1356a34c9a63a675bd5902b6d2fef6f87145847 Mon Sep 17 00:00:00 2001 From: AdarshSantoria Date: Sat, 11 Feb 2023 00:30:33 +0530 Subject: [PATCH 19/88] transfering impl to impl.hpp --- .../methods/randomized_svd/randomized_svd.hpp | 78 +---------------- .../randomized_svd/randomized_svd_impl.hpp | 85 ++++++++++++++++++- 2 files changed, 85 insertions(+), 78 deletions(-) diff --git a/src/mlpack/methods/randomized_svd/randomized_svd.hpp b/src/mlpack/methods/randomized_svd/randomized_svd.hpp index 99646bec7c..652fe3032a 100644 --- a/src/mlpack/methods/randomized_svd/randomized_svd.hpp +++ b/src/mlpack/methods/randomized_svd/randomized_svd.hpp @@ -152,83 +152,7 @@ class RandomizedSVD arma::vec& s, arma::mat& v, const size_t rank, - MatType rowMean) - { - if (iteratedPower == 0) - iteratedPower = rank + 2; - - arma::mat R, Q, Qdata; - - // Apply the centered data matrix to a random matrix, obtaining Q. - if (data.n_cols >= data.n_rows) - { - R = arma::randn(data.n_rows, iteratedPower); - Q = (data.t() * R) - arma::repmat(arma::trans(R.t() * rowMean), - data.n_cols, 1); - } - else - { - R = arma::randn(data.n_cols, iteratedPower); - Q = (data * R) - (rowMean * (arma::ones(1, data.n_cols) * R)); - } - - // Form a matrix Q whose columns constitute a - // well-conditioned basis for the columns of the earlier Q. - if (maxIterations == 0) - { - arma::qr_econ(Q, v, Q); - } - else - { - arma::lu(Q, v, Q); - } - - // Perform normalized power iterations. - for (size_t i = 0; i < maxIterations; ++i) - { - if (data.n_cols >= data.n_rows) - { - Q = (data * Q) - rowMean * (arma::ones(1, data.n_cols) * Q); - arma::lu(Q, v, Q); - Q = (data.t() * Q) - arma::repmat(rowMean.t() * Q, data.n_cols, 1); - } - else - { - Q = (data.t() * Q) - arma::repmat(rowMean.t() * Q, data.n_cols, 1); - arma::lu(Q, v, Q); - Q = (data * Q) - (rowMean * (arma::ones(1, data.n_cols) * Q)); - } - - // Computing the LU decomposition is more efficient than computing the QR - // decomposition, so we only use it in the last iteration, a pivoted QR - // decomposition which renormalizes Q, ensuring that the columns of Q are - // orthonormal. - if (i < (maxIterations - 1)) - { - arma::lu(Q, v, Q); - } - else - { - arma::qr_econ(Q, v, Q); - } - } - - // Do economical singular value decomposition and compute only the - // approximations of the left singular vectors by using the centered data - // applied to Q. - if (data.n_cols >= data.n_rows) - { - Qdata = (data * Q) - rowMean * (arma::ones(1, data.n_cols) * Q); - arma::svd_econ(u, s, v, Qdata); - v = Q * v; - } - else - { - Qdata = (Q.t() * data) - arma::repmat(Q.t() * rowMean, 1, data.n_cols); - arma::svd_econ(u, s, v, Qdata); - u = Q * u; - } - } + MatType rowMean); //! Get the size of the normalized power iterations. size_t IteratedPower() const { return iteratedPower; } diff --git a/src/mlpack/methods/randomized_svd/randomized_svd_impl.hpp b/src/mlpack/methods/randomized_svd/randomized_svd_impl.hpp index 6007048ae7..76569a0993 100644 --- a/src/mlpack/methods/randomized_svd/randomized_svd_impl.hpp +++ b/src/mlpack/methods/randomized_svd/randomized_svd_impl.hpp @@ -51,7 +51,6 @@ inline RandomizedSVD::RandomizedSVD( /* Nothing to do here */ } - inline void RandomizedSVD::Apply(const arma::sp_mat& data, arma::mat& u, arma::vec& s, @@ -76,6 +75,90 @@ inline void RandomizedSVD::Apply(const arma::mat& data, Apply(data, u, s, v, rank, rowMean); } +template +inline void RandomizedSVD::Apply(const MatType& data, + arma::mat& u, + arma::vec& s, + arma::mat& v, + const size_t rank, + MatType rowMean) +{ + if (iteratedPower == 0) + iteratedPower = rank + 2; + + arma::mat R, Q, Qdata; + + // Apply the centered data matrix to a random matrix, obtaining Q. + if (data.n_cols >= data.n_rows) + { + R = arma::randn(data.n_rows, iteratedPower); + Q = (data.t() * R) - arma::repmat(arma::trans(R.t() * rowMean), + data.n_cols, 1); + } + else + { + R = arma::randn(data.n_cols, iteratedPower); + Q = (data * R) - (rowMean * (arma::ones(1, data.n_cols) * R)); + } + + // Form a matrix Q whose columns constitute a + // well-conditioned basis for the columns of the earlier Q. + if (maxIterations == 0) + { + arma::qr_econ(Q, v, Q); + } + else + { + arma::lu(Q, v, Q); + } + + // Perform normalized power iterations. + for (size_t i = 0; i < maxIterations; ++i) + { + if (data.n_cols >= data.n_rows) + { + Q = (data * Q) - rowMean * (arma::ones(1, data.n_cols) * Q); + arma::lu(Q, v, Q); + Q = (data.t() * Q) - arma::repmat(rowMean.t() * Q, data.n_cols, 1); + } + else + { + Q = (data.t() * Q) - arma::repmat(rowMean.t() * Q, data.n_cols, 1); + arma::lu(Q, v, Q); + Q = (data * Q) - (rowMean * (arma::ones(1, data.n_cols) * Q)); + } + + // Computing the LU decomposition is more efficient than computing the QR + // decomposition, so we only use it in the last iteration, a pivoted QR + // decomposition which renormalizes Q, ensuring that the columns of Q are + // orthonormal. + if (i < (maxIterations - 1)) + { + arma::lu(Q, v, Q); + } + else + { + arma::qr_econ(Q, v, Q); + } + } + + // Do economical singular value decomposition and compute only the + // approximations of the left singular vectors by using the centered data + // applied to Q. + if (data.n_cols >= data.n_rows) + { + Qdata = (data * Q) - rowMean * (arma::ones(1, data.n_cols) * Q); + arma::svd_econ(u, s, v, Qdata); + v = Q * v; + } + else + { + Qdata = (Q.t() * data) - arma::repmat(Q.t() * rowMean, 1, data.n_cols); + arma::svd_econ(u, s, v, Qdata); + u = Q * u; + } +} + } // namespace mlpack #endif From 7fe377f2a609982581e16b65f163a398ee67a480 Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Fri, 10 Feb 2023 07:34:02 -0600 Subject: [PATCH 20/88] [R] Remove compilation standard settings on Unix (Close #3407) --- src/mlpack/bindings/R/mlpack/src/Makevars | 1 - src/mlpack/bindings/R/mlpack/src/Makevars.win | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/mlpack/bindings/R/mlpack/src/Makevars b/src/mlpack/bindings/R/mlpack/src/Makevars index 489fe04d78..fd44071bab 100644 --- a/src/mlpack/bindings/R/mlpack/src/Makevars +++ b/src/mlpack/bindings/R/mlpack/src/Makevars @@ -1,3 +1,2 @@ PKG_CXXFLAGS = -DBOOST_MATH_PROMOTE_DOUBLE_POLICY=false -I. $(SHLIB_OPENMP_CXXFLAGS) PKG_LIBS = $(SHLIB_OPENMP_CXXFLAGS) $(LAPACK_LIBS) $(BLAS_LIBS) $(FLIBS) -CXX_STD = CXX14 diff --git a/src/mlpack/bindings/R/mlpack/src/Makevars.win b/src/mlpack/bindings/R/mlpack/src/Makevars.win index cb4f589642..0045749a59 100644 --- a/src/mlpack/bindings/R/mlpack/src/Makevars.win +++ b/src/mlpack/bindings/R/mlpack/src/Makevars.win @@ -1,3 +1,3 @@ PKG_CXXFLAGS = -DBOOST_MATH_PROMOTE_DOUBLE_POLICY=false -I. $(SHLIB_OPENMP_CXXFLAGS) -ftrack-macro-expansion=0 -pipe --param ggc-min-expand=10 --param ggc-min-heapsize=8192 PKG_LIBS = $(SHLIB_OPENMP_CXXFLAGS) $(LAPACK_LIBS) $(BLAS_LIBS) $(FLIBS) -CXX_STD = CXX14 +CXX_STD = CXX17 From e0c7866dd34fe8fdb719e966de8a40ae18e4ea7b Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Sat, 11 Feb 2023 22:31:11 +0530 Subject: [PATCH 21/88] Update HISTORY.md Co-authored-by: Ryan Curtin --- HISTORY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index f7d8b9fd17..aa5162ad1e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,7 +1,7 @@ ### mlpack ?.?.? ###### ????-??-?? - * Update outdated codes in cf.md and datasetmapper.md (#3398). + * Update outdated code in tutorials (#3398). * Bugfix for non-square convolution kernels (#3376). From 18fb167df6e18ec98ea96305ae8e1a7eb13ce50d Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Sat, 11 Feb 2023 22:57:27 +0530 Subject: [PATCH 22/88] Update HISTORY.md --- HISTORY.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 5abe54ee22..83fcf18805 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,8 @@ ### mlpack ?.?.? ###### ????-??-?? + + * Fix code in tutorials (#3401). + * Bugfix for non-square convolution kernels (#3376). * Fix a few missing includes in `` (#3374). From e68b44a3c6a7f047b4f35b681e97f1eca4e8c6e6 Mon Sep 17 00:00:00 2001 From: Aditya Raj <96882869+aadi-raj@users.noreply.github.com> Date: Sun, 12 Feb 2023 05:11:48 +0530 Subject: [PATCH 23/88] Necessary changes in doc. --- doc/tutorials/linear_regression.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/tutorials/linear_regression.md b/doc/tutorials/linear_regression.md index 9a0fb67e66..e8c86f16e2 100644 --- a/doc/tutorials/linear_regression.md +++ b/doc/tutorials/linear_regression.md @@ -339,7 +339,7 @@ file. The class provides one method that performs computation: ```c++ -void Predict(const arma::mat& points, arma::vec& predictions); +void Predict(const arma::mat& points, arma::rowvec& predictions); ``` Once you have generated or loaded a model, you can call this method and pass it @@ -355,7 +355,7 @@ corresponding to each row of the points matrix. using namespace mlpack; arma::mat data; // The dataset itself. -arma::vec responses; // The responses, one row for each row in data. +arma::rowvec responses; // The responses, one row for each row in data. // Regress. LinearRegression lr(data, responses); @@ -400,11 +400,11 @@ LinearRegression lr(); // The dataset we want to predict on; each row is a data point. arma::mat points; // This will store the predictions; one row for each point. -arma::vec predictions; +arma::rowvec predictions; lr.Predict(points, predictions); // Predict. -// Now, the vector 'predictions' will contain the predicted values. +// Now, the row vector 'predictions' will contain the predicted values. ``` ### Setting lambda for ridge regression @@ -419,7 +419,7 @@ used to set a value of lambda: using namespace mlpack; arma::mat data; // The dataset itself. -arma::vec responses; // The responses, one row for each row in data. +arma::rowvec responses; // The responses, one row for each row in data. // Regress, with a lambda of 0.5. LinearRegression lr(data, responses, 0.5); From 555e81417458f46a84b552677903a6ca0fe02211 Mon Sep 17 00:00:00 2001 From: Aditya Raj <96882869+aadi-raj@users.noreply.github.com> Date: Sun, 12 Feb 2023 05:15:20 +0530 Subject: [PATCH 24/88] to avoid confusion as it confused me little bit. --- src/mlpack/methods/linear_regression/linear_regression.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/linear_regression/linear_regression.hpp b/src/mlpack/methods/linear_regression/linear_regression.hpp index 9a0ee221aa..0890e19d2b 100644 --- a/src/mlpack/methods/linear_regression/linear_regression.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression.hpp @@ -39,7 +39,7 @@ class LinearRegression const bool intercept = true); /** - * Creates the model with weighted learning. + * Creates the model with already given weights. * * @param predictors X, matrix of data points. * @param responses y, the measured data for each point in X. From 4738759c711ec13b8c0adf34a9c627ed0270f173 Mon Sep 17 00:00:00 2001 From: Aditya Raj <96882869+aadi-raj@users.noreply.github.com> Date: Sun, 12 Feb 2023 05:22:20 +0530 Subject: [PATCH 25/88] fixed misleading doc --- src/mlpack/core/util/params_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/util/params_impl.hpp b/src/mlpack/core/util/params_impl.hpp index 53e5f5cfdb..54bc5db1ef 100644 --- a/src/mlpack/core/util/params_impl.hpp +++ b/src/mlpack/core/util/params_impl.hpp @@ -45,7 +45,7 @@ inline Params::Params() /** * Return `true` if the specified parameter was given. * - * @param identifier The name of the parameter in question. + * @param key The name of the parameter in question. */ inline bool Params::Has(const std::string& key) const { From ce472b5c31f37c1a82b760d391ba1e1ac1cf9741 Mon Sep 17 00:00:00 2001 From: Aditya Raj <96882869+aadi-raj@users.noreply.github.com> Date: Sun, 12 Feb 2023 05:25:44 +0530 Subject: [PATCH 26/88] style fix --- doc/tutorials/amf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/amf.md b/doc/tutorials/amf.md index 5579b169b2..7801e5d47d 100644 --- a/doc/tutorials/amf.md +++ b/doc/tutorials/amf.md @@ -1,7 +1,7 @@ # Alternating Matrix Factorization tutorial Alternating matrix factorization decomposes a matrix `V` in the form `V ~ WH` -where `W` is called the basis matrix and `H` is called the encoding matrix.. `V` +where `W` is called the basis matrix and `H` is called the encoding matrix. `V` is taken to be of size `n x m` and the obtained `W` is `n x r` and `H` is `r x m`. The size `r` is called the *rank* of the factorization. Factorization is done by alternately calculating `W` and `H` respectively while holding the other From 881635bcb09cfd36618d03d84c7a8d52b18df31a Mon Sep 17 00:00:00 2001 From: Aditya Raj <96882869+aadi-raj@users.noreply.github.com> Date: Sun, 12 Feb 2023 05:31:38 +0530 Subject: [PATCH 27/88] not a necessary fix --- src/mlpack/bindings/python/print_wrapper_py.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/bindings/python/print_wrapper_py.cpp b/src/mlpack/bindings/python/print_wrapper_py.cpp index 130a23a5d7..bd0e517100 100644 --- a/src/mlpack/bindings/python/print_wrapper_py.cpp +++ b/src/mlpack/bindings/python/print_wrapper_py.cpp @@ -438,8 +438,8 @@ void PrintWrapperPY(const std::string& category, cout << endl; indent -= 2; } -} +} // PrintWrapperPY -} // python -} // bindings -} // mlpack +} // namespace python +} // namespace bindings +} // namespace mlpack From d0170c14b3d7de83c68003a666458798ba3e3fed Mon Sep 17 00:00:00 2001 From: Aditya Raj <96882869+aadi-raj@users.noreply.github.com> Date: Sun, 12 Feb 2023 05:38:01 +0530 Subject: [PATCH 28/88] For better understanding of developers(as I have faced difficulty) --- .../bindings/markdown/print_param_table.hpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/mlpack/bindings/markdown/print_param_table.hpp b/src/mlpack/bindings/markdown/print_param_table.hpp index 51d9e9c9cd..dcfa9d3acc 100644 --- a/src/mlpack/bindings/markdown/print_param_table.hpp +++ b/src/mlpack/bindings/markdown/print_param_table.hpp @@ -19,15 +19,15 @@ * Print a table in markdown format that contains * a list of parameters. * - * @param bindingName parameters corresponding to bindingName. - * @param language parameters for a particular language. + * @param bindingName Parameters corresponding to bindingName. + * @param language Parameters for a particular language. * @param params Params object. - * @param headers which headers to print (eg: Name, Default, etc.). - * @param paramsSet to prevent printing a parameter more than once. - * @param onlyHyperParams print only hyper-parameters. - * @param onlyMatrixParams print only matrix-parameters. - * @param onlyInputParams print only input-parameters. - * @param onlyOutputParams print only output-parameters. + * @param headers Which headers to print (eg: Name, Default, etc.). + * @param paramsSet To prevent printing a parameter more than once. + * @param onlyHyperParams If true, print only hyper-parameters. + * @param onlyMatrixParams If true, print only matrix-parameters. + * @param onlyInputParams If true, print only input-parameters. + * @param onlyOutputParams If true, print only output-parameters. */ void PrintParamTable(const std::string& bindingName, const std::string& language, From 3052173a48814951106d5005adb5088b1f994e42 Mon Sep 17 00:00:00 2001 From: Aditya Raj <96882869+aadi-raj@users.noreply.github.com> Date: Sun, 12 Feb 2023 05:41:46 +0530 Subject: [PATCH 29/88] Typo fix --- src/mlpack/methods/svdplusplus/svdplusplus.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/svdplusplus/svdplusplus.hpp b/src/mlpack/methods/svdplusplus/svdplusplus.hpp index 2bffd02e4c..0d3a7d7bf9 100644 --- a/src/mlpack/methods/svdplusplus/svdplusplus.hpp +++ b/src/mlpack/methods/svdplusplus/svdplusplus.hpp @@ -21,7 +21,7 @@ namespace mlpack { /** - * SVD++ is a matrix decomposition tenique used in collaborative filtering. + * SVD++ is a matrix decomposition technique used in collaborative filtering. * SVD++ is similar to BiasSVD, but it is a more expressive model because * SVD++ also models implicit feedback. SVD++ outputs user/item latent * vectors, user/item bias, and item vectors with regard to implicit feedback. From 3de2a1a799ef51119b9e50ebf72e3d508fb18442 Mon Sep 17 00:00:00 2001 From: Aditya Raj <96882869+aadi-raj@users.noreply.github.com> Date: Sun, 12 Feb 2023 05:50:03 +0530 Subject: [PATCH 30/88] fixed docs --- src/mlpack/core/util/param.hpp | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/mlpack/core/util/param.hpp b/src/mlpack/core/util/param.hpp index d6faad82a4..213982e1ab 100644 --- a/src/mlpack/core/util/param.hpp +++ b/src/mlpack/core/util/param.hpp @@ -42,12 +42,6 @@ * should be present per binding. BINDING_NAME should be set before calling * this. * - * @see mlpack::IO, PARAM_FLAG(), PARAM_INT_IN(), PARAM_DOUBLE_IN(), - * PARAM_STRING_IN(), PARAM_VECTOR_IN(), PARAM_INT_OUT(), PARAM_DOUBLE_OUT(), - * PARAM_VECTOR_OUT(), PARAM_INT_IN_REQ(), PARAM_DOUBLE_IN_REQ(), - * PARAM_STRING_IN_REQ(), PARAM_VECTOR_IN_REQ(), PARAM_INT_OUT_REQ(), - * PARAM_DOUBLE_OUT_REQ(), PARAM_VECTOR_OUT_REQ(), PARAM_STRING_OUT_REQ(). - * * @param NAME User-friendly name. */ #ifdef __COUNTER__ @@ -179,17 +173,6 @@ * printing macros like PRINT_PARAM_STRING() or PRINT_DATASET() or others * here---it will cause problems. * @param ALIAS An alias for the parameter (one letter). - * - * @see mlpack::IO, BINDING_USER_NAME(), BINDING_SHORT_DESC(), - * BINDING_LONG_DESC(), BINDING_EXAMPLE() and BINDING_SEE_ALSO(). - * - * @bug - * The __COUNTER__ variable is used in most cases to guarantee a unique global - * identifier for options declared using the PARAM_*() macros. However, not all - * compilers have this support--most notably, gcc < 4.3. In that case, the - * __LINE__ macro is used as an attempt to get a unique global identifier, but - * collisions are still possible, and they produce bizarre error messages. See - * https://github.com/mlpack/mlpack/issues/100 for more information. */ #define PARAM_FLAG(ID, DESC, ALIAS) \ PARAM_IN(bool, ID, DESC, ALIAS, false, false); From 7c961c50d59eef1d532c6cca40c0f040c14814cf Mon Sep 17 00:00:00 2001 From: Aditya Raj <96882869+aadi-raj@users.noreply.github.com> Date: Sun, 12 Feb 2023 05:54:10 +0530 Subject: [PATCH 31/88] changed datatype --- src/mlpack/bindings/python/print_wrapper_py.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/bindings/python/print_wrapper_py.cpp b/src/mlpack/bindings/python/print_wrapper_py.cpp index bd0e517100..c733e2c2a3 100644 --- a/src/mlpack/bindings/python/print_wrapper_py.cpp +++ b/src/mlpack/bindings/python/print_wrapper_py.cpp @@ -64,7 +64,7 @@ void PrintWrapperPY(const std::string& category, } // Import different mlpack programs that are to be wrapped. - for(int i=0; i Date: Sun, 12 Feb 2023 12:55:13 +0530 Subject: [PATCH 32/88] Added preprocessing part and tests --- .../quic_svd_method.hpp | 17 ++---- src/mlpack/tests/cf_test.cpp | 61 +++++++++++++++++++ 2 files changed, 67 insertions(+), 11 deletions(-) diff --git a/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp b/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp index e199359f2d..330b8da278 100644 --- a/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp +++ b/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp @@ -52,17 +52,17 @@ class QuicSVDPolicy * Apply Collaborative Filtering to the provided data set using the * quic SVD. * - * @param data Data matrix: dense matrix (coordinate lists) + * @param * (data) Data matrix: dense matrix (coordinate lists) * or sparse matrix(cleaned). - * @param * (cleanedData) item user table in form of sparse matrix. + * @param cleanedData item user table in form of sparse matrix. * @param * (rank) Rank parameter for matrix factorization. * @param * (maxIterations) Maximum number of iterations. * @param * (minResidue) Residue required to terminate. * @param * (mit) Whether to terminate only when maxIterations is reached. */ template - void Apply(const MatType& data, - const arma::sp_mat& /* cleanedData */, + void Apply(const MatType& /* data */, + const arma::sp_mat& cleanedData, const size_t /* rank */, const size_t /* maxIterations */, const double /* minResidue */, @@ -70,17 +70,12 @@ class QuicSVDPolicy { arma::mat sigma; + arma::mat data(cleanedData); + // Do singular value decomposition using the quic SVD algorithm. QUIC_SVD quicsvd; quicsvd.Apply(data, w, h, sigma); - if (data.n_cols < data.n_rows) - { - arma::mat tempMat = w; - w = h; - h = tempMat; - } - // Take transpose of the matrix h as required by CF class. h = arma::trans(h); } diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index b876b3316c..64708304a2 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -567,6 +567,15 @@ TEST_CASE("CFGetRecommendationsAllUsersSVDPPTest", "[CFTest]") GetRecommendationsAllUsers(); } +/** + * Make sure that correct number of recommendations are generated when query + * set for Quic SVD method. + */ +TEST_CASE("CFGetRecommendationsAllUsersQSVDTest", "[CFTest]") +{ + GetRecommendationsAllUsers(); +} + /** * Make sure that the recommendations are generated for queried users only * for randomized SVD. @@ -639,6 +648,15 @@ TEST_CASE("CFGetRecommendationsQueriedUserSVDPPTest", "[CFTest]") GetRecommendationsQueriedUser(); } +/** + * Make sure that the recommendations are generated for queried users only + * for Quic SVD method. + */ +TEST_CASE("CFGetRecommendationsQueriedUserQSVDTest", "[CFTest]") +{ + GetRecommendationsQueriedUser(); +} + /** * Make sure recommendations that are generated are reasonably accurate * for randomized SVD. @@ -775,6 +793,15 @@ TEST_CASE("CFPredictSVDPPTest", "[CFTest]") CFPredict(); } +/** + * Make sure that Predict() is returning reasonable results for Quic SVD + * method. + */ +TEST_CASE("CFPredictQSVDTest", "[CFTest]") +{ + CFPredict(); +} + // Compare batch Predict() and individual Predict() for randomized SVD. TEST_CASE("CFBatchPredictRandSVDTest", "[CFTest]") { @@ -827,6 +854,13 @@ TEST_CASE("CFBatchPredictSVDPPTest", "[CFTest]") BatchPredict(); } +// Compare batch Predict() and individual Predict() for +// Quic SVD method. +TEST_CASE("CFBatchPredictQSVDTest", "[CFTest]") +{ + BatchPredict(); +} + /** * Make sure we can train an already-trained model and it works okay for * randomized SVD. @@ -907,6 +941,16 @@ TEST_CASE("TrainSVDPPTest", "[CFTest]") TrainWithCoordinateList(decomposition); } +/** + * Make sure we can train an already-trained model and it works okay for + * Quic SVD method. + */ +TEST_CASE("TrainQSVDTest", "[CFTest]") +{ + QuicSVDPolicy decomposition; + TrainWithCoordinateList(decomposition); +} + /** * Make sure we can train a model after using the empty constructor when * using randomized SVD. @@ -961,6 +1005,15 @@ TEST_CASE("EmptyConstructorTrainSVDIncompleteTest", "[CFTest]") EmptyConstructorTrain(); } +/** + * Make sure we can train a model after using the empty constructor when + * using Quic SVD method. + */ +TEST_CASE("EmptyConstructorTrainQSVDTest", "[CFTest]") +{ + EmptyConstructorTrain(); +} + /** * Ensure we can load and save the CF model using randomized SVD policy. */ @@ -1001,6 +1054,14 @@ TEST_CASE("SerializationSVDIncompleteTest", "[CFTest]") Serialization(); } +/** + * Ensure we can load and save the CF model using Quic SVD Incremental. +// */ +TEST_CASE("SerializationQSVDTest", "[CFTest]") +{ + Serialization(); +} + /** * Make sure that Predict() is returning reasonable results for NMF and * OverallMeanNormalization. From 3315aff9865d13d9d49b58829f29b5c13b02f971 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Sun, 12 Feb 2023 13:00:16 +0530 Subject: [PATCH 33/88] Add comment --- src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp b/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp index 330b8da278..bf0abee51e 100644 --- a/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp +++ b/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp @@ -70,6 +70,7 @@ class QuicSVDPolicy { arma::mat sigma; + // Preprocessed data converted to mat format arma::mat data(cleanedData); // Do singular value decomposition using the quic SVD algorithm. From b32355d0a23684e2fcafd80c4e56599424b8c31f Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Sun, 12 Feb 2023 13:05:06 +0530 Subject: [PATCH 34/88] Update HISTORY.md --- HISTORY.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 36232c4eeb..73079b0a80 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,8 @@ ### mlpack ?.?.? ###### ????-??-?? + + * Add QUIC_SVD method in CF (#3404). + * Bugfix for non-square convolution kernels (#3376). * Fix a few missing includes in `` (#3374). From 2b7bb737f31d69f8ff4cb57af5a09c66d3bde838 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Sun, 12 Feb 2023 13:10:14 +0530 Subject: [PATCH 35/88] fix typo --- src/mlpack/tests/cf_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index 64708304a2..cf39fec15b 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -1055,8 +1055,8 @@ TEST_CASE("SerializationSVDIncompleteTest", "[CFTest]") } /** - * Ensure we can load and save the CF model using Quic SVD Incremental. -// */ + * Ensure we can load and save the CF model using Quic SVD Policy. + */ TEST_CASE("SerializationQSVDTest", "[CFTest]") { Serialization(); From 24c04612086f08c691b541cd9cda2186b1beba33 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Sun, 12 Feb 2023 13:35:07 +0530 Subject: [PATCH 36/88] Revert Changes --- HISTORY.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 83fcf18805..5abe54ee22 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,8 +1,5 @@ ### mlpack ?.?.? ###### ????-??-?? - - * Fix code in tutorials (#3401). - * Bugfix for non-square convolution kernels (#3376). * Fix a few missing includes in `` (#3374). From 6309e47a9f2c5f1fe571efb1d3f0bf9bda2b935a Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Sun, 12 Feb 2023 13:37:06 +0530 Subject: [PATCH 37/88] Revert Changes --- HISTORY.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 73079b0a80..36232c4eeb 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,8 +1,5 @@ ### mlpack ?.?.? ###### ????-??-?? - - * Add QUIC_SVD method in CF (#3404). - * Bugfix for non-square convolution kernels (#3376). * Fix a few missing includes in `` (#3374). From e338815b51be3fa2fcd174ca37f5d64722f0d11c Mon Sep 17 00:00:00 2001 From: AdarshSantoria Date: Sun, 12 Feb 2023 17:35:03 +0530 Subject: [PATCH 38/88] Add block krylov svd method for cf --- .../block_krylov_svd_method.hpp | 170 ++++++++++++++++++ .../decomposition_policies.hpp | 1 + src/mlpack/tests/cf_test.cpp | 61 +++++++ 3 files changed, 232 insertions(+) create mode 100644 src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp diff --git a/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp b/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp new file mode 100644 index 0000000000..cae19a18c7 --- /dev/null +++ b/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp @@ -0,0 +1,170 @@ +/** + * @file methods/cf/decomposition_policies/block_krylov_svd_method.hpp + * @author Adarsh Santoria + * + * Implementation of the block krylov svd method for use in + * Collaborative Fitlering. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_METHODS_CF_DECOMPOSITION_POLICIES_BLOCK_KRYLOV_SVD_METHOD_HPP +#define MLPACK_METHODS_CF_DECOMPOSITION_POLICIES_BLOCK_KRYLOV_SVD_METHOD_HPP + +#include +#include + +namespace mlpack { + +/** + * Implementation of the BLOCK-KRYLOV-SVD policy to act as a wrapper when + * accessing BLOCK KRYLOV SVD from within CFType. + * + * An example of how to use BlockKrylovSVDPolicy in CF is shown below: + * + * @code + * extern arma::mat data; // data is a (user, item, rating) table. + * // Users for whom recommendations are generated. + * extern arma::Col users; + * arma::Mat recommendations; // Resulting recommendations. + * + * CFType cf(data); + * + * // Generate 10 recommendations for all users. + * cf.GetRecommendations(10, recommendations); + * @endcode + */ +class BlockKrylovSVDPolicy +{ + public: + /** + * Use block krylov SVD method to perform collaborative filtering + */ + BlockKrylovSVDPolicy() + { + /* Nothing to do here */ + } + + /** + * Apply Collaborative Filtering to the provided data set using the + * block krylov SVD. + * + * @param * (data) Data matrix: dense matrix (coordinate lists) + * or sparse matrix(cleaned). + * @param cleanedData item user table in form of sparse matrix. + * @param rank Rank parameter for matrix factorization. + * @param * (maxIterations) Maximum number of iterations. + * @param * (minResidue) Residue required to terminate. + * @param * (mit) Whether to terminate only when maxIterations is reached. + */ + template + void Apply(const MatType& /* data */, + const arma::sp_mat& cleanedData, + const size_t rank, + const size_t /* maxIterations */, + const double /* minResidue */, + const bool /* mit */) + { + arma::vec sigma; + + // Preprocessed data converted to mat format + arma::mat data(cleanedData); + + // Do singular value decomposition using the block krylov SVD algorithm. + RandomizedBlockKrylovSVD blockkylovsvd; + blockkylovsvd.Apply(data, w, sigma, h, rank); + + // Take transpose of the matrix h as required by CF class. + h = arma::trans(h); + } + + /** + * Return predicted rating given user ID and item ID. + * + * @param user User ID. + * @param item Item ID. + */ + double GetRating(const size_t user, const size_t item) const + { + double rating = arma::as_scalar(w.row(item) * h.col(user)); + return rating; + } + + /** + * Get predicted ratings for a user. + * + * @param user User ID. + * @param rating Resulting rating vector. + */ + void GetRatingOfUser(const size_t user, arma::vec& rating) const + { + rating = w * h.col(user); + } + + /** + * Get the neighborhood and corresponding similarities for a set of users. + * + * @tparam NeighborSearchPolicy The policy to perform neighbor search. + * + * @param users Users whose neighborhood is to be computed. + * @param numUsersForSimilarity The number of neighbors returned for + * each user. + * @param neighborhood Neighbors represented by user IDs. + * @param similarities Similarity between each user and each of its + * neighbors. + */ + template + void GetNeighborhood(const arma::Col& users, + const size_t numUsersForSimilarity, + arma::Mat& neighborhood, + arma::mat& similarities) const + { + // We want to avoid calculating the full rating matrix, so we will do + // nearest neighbor search only on the H matrix, using the observation that + // if the rating matrix X = W*H, then d(X.col(i), X.col(j)) = d(W H.col(i), + // W H.col(j)). This can be seen as nearest neighbor search on the H + // matrix with the Mahalanobis distance where M^{-1} = W^T W. So, we'll + // decompose M^{-1} = L L^T (the Cholesky decomposition), and then multiply + // H by L^T. Then we can perform nearest neighbor search. + arma::mat l = arma::chol(w.t() * w); + arma::mat stretchedH = l * h; // Due to the Armadillo API, l is L^T. + + // Temporarily store feature vector of queried users. + arma::mat query(stretchedH.n_rows, users.n_elem); + // Select feature vectors of queried users. + for (size_t i = 0; i < users.n_elem; ++i) + query.col(i) = stretchedH.col(users(i)); + + NeighborSearchPolicy neighborSearch(stretchedH); + neighborSearch.Search( + query, numUsersForSimilarity, neighborhood, similarities); + } + + //! Get the Item Matrix. + const arma::mat& W() const { return w; } + //! Get the User Matrix. + const arma::mat& H() const { return h; } + + /** + * Serialization. + */ + template + void serialize(Archive& ar, const uint32_t /* version */) + { + ar(CEREAL_NVP(w)); + ar(CEREAL_NVP(h)); + } + + private: + //! Item matrix. + arma::mat w; + //! User matrix. + arma::mat h; +}; + +} // namespace mlpack + +#endif \ No newline at end of file diff --git a/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp b/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp index 731b8db8d1..ca3df3af0f 100644 --- a/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp +++ b/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp @@ -20,5 +20,6 @@ #include "svd_complete_method.hpp" #include "svd_incomplete_method.hpp" #include "svdplusplus_method.hpp" +#include "block_krylov_svd_method.hpp" #endif diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index b876b3316c..ab21fe2b4d 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -567,6 +567,15 @@ TEST_CASE("CFGetRecommendationsAllUsersSVDPPTest", "[CFTest]") GetRecommendationsAllUsers(); } +/** + * Make sure that correct number of recommendations are generated when query + * set for Block Krylov SVD method. + */ +TEST_CASE("CFGetRecommendationsAllUsersBKSVDTest", "[CFTest]") +{ + GetRecommendationsAllUsers(); +} + /** * Make sure that the recommendations are generated for queried users only * for randomized SVD. @@ -639,6 +648,15 @@ TEST_CASE("CFGetRecommendationsQueriedUserSVDPPTest", "[CFTest]") GetRecommendationsQueriedUser(); } +/** + * Make sure that the recommendations are generated for queried users only + * for Block Krylov SVD method. + */ +TEST_CASE("CFGetRecommendationsQueriedUserBKSVDTest", "[CFTest]") +{ + GetRecommendationsQueriedUser(); +} + /** * Make sure recommendations that are generated are reasonably accurate * for randomized SVD. @@ -775,6 +793,15 @@ TEST_CASE("CFPredictSVDPPTest", "[CFTest]") CFPredict(); } +/** + * Make sure that Predict() is returning reasonable results for Block Krylov SVD + * method. + */ +TEST_CASE("CFPredictBKSVDTest", "[CFTest]") +{ + CFPredict(); +} + // Compare batch Predict() and individual Predict() for randomized SVD. TEST_CASE("CFBatchPredictRandSVDTest", "[CFTest]") { @@ -827,6 +854,13 @@ TEST_CASE("CFBatchPredictSVDPPTest", "[CFTest]") BatchPredict(); } +// Compare batch Predict() and individual Predict() for +// Block Krylov SVD method. +TEST_CASE("CFBatchPredictBKSVDTest", "[CFTest]") +{ + BatchPredict(); +} + /** * Make sure we can train an already-trained model and it works okay for * randomized SVD. @@ -907,6 +941,16 @@ TEST_CASE("TrainSVDPPTest", "[CFTest]") TrainWithCoordinateList(decomposition); } +/** + * Make sure we can train an already-trained model and it works okay for + * Block Krylov SVD method. + */ +TEST_CASE("TrainBKSVDTest", "[CFTest]") +{ + BlockKrylovSVDPolicy decomposition; + TrainWithCoordinateList(decomposition); +} + /** * Make sure we can train a model after using the empty constructor when * using randomized SVD. @@ -961,6 +1005,15 @@ TEST_CASE("EmptyConstructorTrainSVDIncompleteTest", "[CFTest]") EmptyConstructorTrain(); } +/** + * Make sure we can train a model after using the empty constructor when + * using Block Krylov SVD method. + */ +TEST_CASE("EmptyConstructorTrainBKSVDTest", "[CFTest]") +{ + EmptyConstructorTrain(); +} + /** * Ensure we can load and save the CF model using randomized SVD policy. */ @@ -1001,6 +1054,14 @@ TEST_CASE("SerializationSVDIncompleteTest", "[CFTest]") Serialization(); } +/** + * Ensure we can load and save the CF model using Block Krylov SVD. + */ +TEST_CASE("SerializationBKSVDTest", "[CFTest]") +{ + Serialization(); +} + /** * Make sure that Predict() is returning reasonable results for NMF and * OverallMeanNormalization. From b4023ba22983e81e6275b0fe95d848057b31a170 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Sun, 12 Feb 2023 17:40:16 +0530 Subject: [PATCH 39/88] Update cf.md --- doc/tutorials/cf.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/tutorials/cf.md b/doc/tutorials/cf.md index aa3c4fb57e..5749d99957 100644 --- a/doc/tutorials/cf.md +++ b/doc/tutorials/cf.md @@ -279,6 +279,7 @@ alternating least squares update rules). These include: - `BiasSVDPolicy` - `SVDPlusPlusPolicy` - `RandomizedSVDPolicy` + - `BlockKrylovSVDPolicy` The `AMF` class has many other possibilities than those listed here; it is a framework for alternating matrix factorization techniques. See the `AMF` class From 7aa7c9e9e5c36c0fcf2025012494bb9153cf104a Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Sun, 12 Feb 2023 23:42:03 +0530 Subject: [PATCH 40/88] Update HISTORY.md --- HISTORY.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 3976c0d6d3..e71029921c 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,12 @@ ### mlpack ?.?.? ###### ????-??-?? + * Add Block Krylov SVD method in CF (#3413). + + * Add QUIC_SVD method in CF (#3404). + + * Fix code in tutorials (#3401). + * Update outdated code in tutorials (#3398). * Bugfix for non-square convolution kernels (#3376). From 175747d598e8696a1e4d750189cbc848ce1c9e33 Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Sun, 12 Feb 2023 12:51:28 -0600 Subject: [PATCH 41/88] [R] Remove configure script as sunos is no longer an issue --- src/mlpack/bindings/R/mlpack/configure | 8 -------- 1 file changed, 8 deletions(-) delete mode 100755 src/mlpack/bindings/R/mlpack/configure diff --git a/src/mlpack/bindings/R/mlpack/configure b/src/mlpack/bindings/R/mlpack/configure deleted file mode 100755 index 608e27d17e..0000000000 --- a/src/mlpack/bindings/R/mlpack/configure +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -if test `uname` = "SunOS" ; -then -sed '1 s/$/ -ftrack-macro-expansion=0 -pipe --param ggc-min-expand=10 --param ggc-min-heapsize=8192/' ./src/Makevars > ./src/Makevars.tmp && cat ./src/Makevars.tmp > ./src/Makevars && rm ./src/Makevars.tmp -fi - -exit 0 From e3bd456c7f306bfd0baaed4917f1e49fac5becec Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Mon, 13 Feb 2023 01:20:35 +0530 Subject: [PATCH 42/88] Update HISTORY.md Co-authored-by: Ryan Curtin --- HISTORY.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index e71029921c..9739961e53 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -5,9 +5,7 @@ * Add QUIC_SVD method in CF (#3404). - * Fix code in tutorials (#3401). - - * Update outdated code in tutorials (#3398). + * Update outdated code in tutorials (#3398, #3401). * Bugfix for non-square convolution kernels (#3376). From da0fdb47d504d23049a1eed4881073bb7b31d67c Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Mon, 13 Feb 2023 01:20:52 +0530 Subject: [PATCH 43/88] Update HISTORY.md Co-authored-by: Ryan Curtin --- HISTORY.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 9739961e53..2ed659a75c 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,9 +1,7 @@ ### mlpack ?.?.? ###### ????-??-?? - * Add Block Krylov SVD method in CF (#3413). - - * Add QUIC_SVD method in CF (#3404). + * Add CF decomposition methods: `QUIC_SVDPolicy` and `BlockKrylovSVDPolicy` (#3413, #3404). * Update outdated code in tutorials (#3398, #3401). From 2ac83b8b2e765c04ef9b36b2939fa76e15770024 Mon Sep 17 00:00:00 2001 From: Benson Muite Date: Mon, 13 Feb 2023 10:37:12 +0300 Subject: [PATCH 44/88] Update date --- LICENSE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE.txt b/LICENSE.txt index e0d5187dbb..cb1cee1aec 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -3,7 +3,7 @@ can redistribute the library and/or modify it under the terms of the 3-clause BSD license. The text of the 3-clause BSD license is contained below. ---- -Copyright (c) 2007-2022, mlpack contributors (see COPYRIGHT.txt) +Copyright (c) 2007-2023, mlpack contributors (see COPYRIGHT.txt) All rights reserved. Redistribution and use of mlpack in source and binary forms, with or without From 2eff5e056fcfccc8274f1bd7904b935e11083f77 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Mon, 13 Feb 2023 22:02:09 +0530 Subject: [PATCH 45/88] Update src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp Co-authored-by: Ryan Curtin --- .../cf/decomposition_policies/block_krylov_svd_method.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp b/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp index cae19a18c7..8889bf957e 100644 --- a/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp +++ b/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp @@ -74,7 +74,7 @@ class BlockKrylovSVDPolicy arma::mat data(cleanedData); // Do singular value decomposition using the block krylov SVD algorithm. - RandomizedBlockKrylovSVD blockkylovsvd; + RandomizedBlockKrylovSVD blockkrylovsvd; blockkylovsvd.Apply(data, w, sigma, h, rank); // Take transpose of the matrix h as required by CF class. From 7e9e8de0e14296ed8390c078ac4d7d05748437f3 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Mon, 13 Feb 2023 22:02:56 +0530 Subject: [PATCH 46/88] fix typo --- .../cf/decomposition_policies/block_krylov_svd_method.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp b/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp index 8889bf957e..d924ba8ec6 100644 --- a/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp +++ b/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp @@ -75,7 +75,7 @@ class BlockKrylovSVDPolicy // Do singular value decomposition using the block krylov SVD algorithm. RandomizedBlockKrylovSVD blockkrylovsvd; - blockkylovsvd.Apply(data, w, sigma, h, rank); + blockkrylovsvd.Apply(data, w, sigma, h, rank); // Take transpose of the matrix h as required by CF class. h = arma::trans(h); @@ -167,4 +167,4 @@ class BlockKrylovSVDPolicy } // namespace mlpack -#endif \ No newline at end of file +#endif From 0cf1d1efcdd5072ada75c9cce80a0232498966d6 Mon Sep 17 00:00:00 2001 From: Aditya Raj <96882869+aadi-raj@users.noreply.github.com> Date: Mon, 13 Feb 2023 22:13:34 +0530 Subject: [PATCH 47/88] style fix Co-authored-by: Ryan Curtin --- src/mlpack/bindings/python/print_wrapper_py.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/bindings/python/print_wrapper_py.cpp b/src/mlpack/bindings/python/print_wrapper_py.cpp index c733e2c2a3..3ed5bc2976 100644 --- a/src/mlpack/bindings/python/print_wrapper_py.cpp +++ b/src/mlpack/bindings/python/print_wrapper_py.cpp @@ -64,7 +64,7 @@ void PrintWrapperPY(const std::string& category, } // Import different mlpack programs that are to be wrapped. - for(size_t i=0; i Date: Mon, 13 Feb 2023 22:18:31 +0530 Subject: [PATCH 48/88] Reverting changes --- src/mlpack/methods/linear_regression/linear_regression.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/linear_regression/linear_regression.hpp b/src/mlpack/methods/linear_regression/linear_regression.hpp index 0890e19d2b..9a0ee221aa 100644 --- a/src/mlpack/methods/linear_regression/linear_regression.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression.hpp @@ -39,7 +39,7 @@ class LinearRegression const bool intercept = true); /** - * Creates the model with already given weights. + * Creates the model with weighted learning. * * @param predictors X, matrix of data points. * @param responses y, the measured data for each point in X. From d11cb3fc0b57c24e822b08c1ec38d4a48b1400bd Mon Sep 17 00:00:00 2001 From: AdarshSantoria Date: Tue, 14 Feb 2023 02:04:16 +0530 Subject: [PATCH 49/88] fix quicsvd bug and add template tests --- .../quic_svd_method.hpp | 11 +- src/mlpack/methods/quic_svd/quic_svd.hpp | 2 +- src/mlpack/methods/quic_svd/quic_svd_impl.hpp | 3 + src/mlpack/tests/cf_test.cpp | 715 ++---------------- 4 files changed, 90 insertions(+), 641 deletions(-) diff --git a/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp b/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp index bf0abee51e..33d6cc76c3 100644 --- a/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp +++ b/src/mlpack/methods/cf/decomposition_policies/quic_svd_method.hpp @@ -23,7 +23,7 @@ namespace mlpack { * Implementation of the QUIC-SVD policy to act as a wrapper when * accessing Quic SVD from within CFType. * - * An example of how to use QuicSVDPolicy in CF is shown below: + * An example of how to use QUIC_SVDPolicy in CF is shown below: * * @code * extern arma::mat data; // data is a (user, item, rating) table. @@ -31,19 +31,19 @@ namespace mlpack { * extern arma::Col users; * arma::Mat recommendations; // Resulting recommendations. * - * CFType cf(data); + * CFType cf(data); * * // Generate 10 recommendations for all users. * cf.GetRecommendations(10, recommendations); * @endcode */ -class QuicSVDPolicy +class QUIC_SVDPolicy { public: /** * Use quic SVD method to perform collaborative filtering */ - QuicSVDPolicy() + QUIC_SVDPolicy() { /* Nothing to do here */ } @@ -77,6 +77,9 @@ class QuicSVDPolicy QUIC_SVD quicsvd; quicsvd.Apply(data, w, h, sigma); + // Sigma matrix is multiplied to w. + w = w * sigma; + // Take transpose of the matrix h as required by CF class. h = arma::trans(h); } diff --git a/src/mlpack/methods/quic_svd/quic_svd.hpp b/src/mlpack/methods/quic_svd/quic_svd.hpp index 0e0e40896e..d3c8d976a5 100644 --- a/src/mlpack/methods/quic_svd/quic_svd.hpp +++ b/src/mlpack/methods/quic_svd/quic_svd.hpp @@ -43,7 +43,7 @@ namespace mlpack { * const double delta = 0.1 // Lower error bound for Monte Carlo estimate. * * // Make a QuicSVD object. - * QuicSVD qSVD(); + * QUIC_SVD qSVD(); * * arma::mat u, v, sigma; // Matrices for the factors. data = u * sigma * v.t() * diff --git a/src/mlpack/methods/quic_svd/quic_svd_impl.hpp b/src/mlpack/methods/quic_svd/quic_svd_impl.hpp index 3bbc551f9f..d44c88c1b9 100644 --- a/src/mlpack/methods/quic_svd/quic_svd_impl.hpp +++ b/src/mlpack/methods/quic_svd/quic_svd_impl.hpp @@ -82,6 +82,9 @@ inline void QUIC_SVD::ExtractSVD(const arma::mat& dataset, arma::vec sigmaBar; arma::svd(uBar, sigmaBar, vBar, projectedMatSquared); + // vBar was tarnsposed of vBar + vBar = arma::trans(vBar); + // Calculate the approximate SVD of the original matrix, using the SVD of the // squared projected matrix. v = basis * vBar; diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index cf39fec15b..db6232c81e 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -496,230 +496,37 @@ void Serialization() /** * Make sure that correct number of recommendations are generated when query - * set for randomized SVD. + * set for all methods. */ -TEST_CASE("CFGetRecommendationsAllUsersRandSVDTest", "[CFTest]") +TEMPLATE_TEST_CASE("CFGetRecommendationsAllUsersTest", "[CFTest]", + RandomizedSVDPolicy, RegSVDPolicy, BatchSVDPolicy, NMFPolicy, + SVDCompletePolicy, SVDIncompletePolicy, BiasSVDPolicy, SVDPlusPlusPolicy, + QUIC_SVDPolicy) { - GetRecommendationsAllUsers(); + GetRecommendationsAllUsers(); } /** - * Make sure that correct number of recommendations are generated when query - * set for regularized SVD. + * Make sure that the recommendations are generated for queried users + * for all methods. */ -TEST_CASE("CFGetRecommendationsAllUsersRegSVDTest", "[CFTest]") +TEMPLATE_TEST_CASE("CFGetRecommendationsQueriedUsersTest", "[CFTest]", + RandomizedSVDPolicy, RegSVDPolicy, BatchSVDPolicy, NMFPolicy, + SVDCompletePolicy, SVDIncompletePolicy, BiasSVDPolicy, SVDPlusPlusPolicy, + QUIC_SVDPolicy) { - GetRecommendationsAllUsers(); -} - -/** - * Make sure that correct number of recommendations are generated when query - * set for Batch SVD. - */ - -TEST_CASE("CFGetRecommendationsAllUsersBatchSVDTest", "[CFTest]") -{ - GetRecommendationsAllUsers(); -} - -/** - * Make sure that correct number of recommendations are generated when query - * set for NMF. - */ -TEST_CASE("CFGetRecommendationsAllUsersNMFTest", "[CFTest]") -{ - GetRecommendationsAllUsers(); -} - -/** - * Make sure that correct number of recommendations are generated when query - * set for SVD Complete Incremental method. - */ -TEST_CASE("CFGetRecommendationsAllUsersSVDCompleteTest", "[CFTest]") -{ - GetRecommendationsAllUsers(); -} - -/** - * Make sure that correct number of recommendations are generated when query - * set for SVD Incomplete Incremental method. - */ -TEST_CASE("CFGetRecommendationsAllUsersSVDIncompleteTest", "[CFTest]") -{ - GetRecommendationsAllUsers(); -} - -/** - * Make sure that correct number of recommendations are generated when query - * set for Bias SVD method. - */ -TEST_CASE("CFGetRecommendationsAllUsersBiasSVDTest", "[CFTest]") -{ - GetRecommendationsAllUsers(); -} - -/** - * Make sure that correct number of recommendations are generated when query - * set for SVDPlusPlus method. - */ -TEST_CASE("CFGetRecommendationsAllUsersSVDPPTest", "[CFTest]") -{ - GetRecommendationsAllUsers(); -} - -/** - * Make sure that correct number of recommendations are generated when query - * set for Quic SVD method. - */ -TEST_CASE("CFGetRecommendationsAllUsersQSVDTest", "[CFTest]") -{ - GetRecommendationsAllUsers(); -} - -/** - * Make sure that the recommendations are generated for queried users only - * for randomized SVD. - */ -TEST_CASE("CFGetRecommendationsQueriedUserRandSVDTest", "[CFTest]") -{ - GetRecommendationsQueriedUser(); -} - -/** - * Make sure that the recommendations are generated for queried users only - * for regularized SVD. - */ -TEST_CASE("CFGetRecommendationsQueriedUserRegSVDTest", "[CFTest]") -{ - GetRecommendationsQueriedUser(); -} - -/** - * Make sure that the recommendations are generated for queried users only - * for batch SVD. - */ -TEST_CASE("CFGetRecommendationsQueriedUserBatchSVDTest", "[CFTest]") -{ - GetRecommendationsQueriedUser(); -} - -/** - * Make sure that the recommendations are generated for queried users only - * for NMF. - */ -TEST_CASE("CFGetRecommendationsQueriedUserNMFTest", "[CFTest]") -{ - GetRecommendationsQueriedUser(); -} - -/** - * Make sure that the recommendations are generated for queried users only - * for SVD Complete Incremental method. - */ -TEST_CASE("CFGetRecommendationsQueriedUserSVDCompleteTest", "[CFTest]") -{ - GetRecommendationsQueriedUser(); -} - -/** - * Make sure that the recommendations are generated for queried users only - * for SVD Incomplete Incremental method. - */ -TEST_CASE("CFGetRecommendationsQueriedUserSVDIncompleteTest", "[CFTest]") -{ - GetRecommendationsQueriedUser(); -} - -/** - * Make sure that the recommendations are generated for queried users only - * for Bias SVD method. - */ -TEST_CASE("CFGetRecommendationsQueriedUserBiasSVDTest", "[CFTest]") -{ - GetRecommendationsQueriedUser(); -} - -/** - * Make sure that the recommendations are generated for queried users only - * for SVDPlusPlus method. - */ -TEST_CASE("CFGetRecommendationsQueriedUserSVDPPTest", "[CFTest]") -{ - GetRecommendationsQueriedUser(); -} - -/** - * Make sure that the recommendations are generated for queried users only - * for Quic SVD method. - */ -TEST_CASE("CFGetRecommendationsQueriedUserQSVDTest", "[CFTest]") -{ - GetRecommendationsQueriedUser(); + GetRecommendationsQueriedUser(); } /** * Make sure recommendations that are generated are reasonably accurate - * for randomized SVD. + * for all methods except SVDPlusPlus method. */ -TEST_CASE("RecommendationAccuracyRandSVDTest", "[CFTest]") +TEMPLATE_TEST_CASE("RecommendationAccuracyTest", "[CFTest]", + RandomizedSVDPolicy, RegSVDPolicy, BatchSVDPolicy, NMFPolicy, + SVDCompletePolicy, SVDIncompletePolicy, BiasSVDPolicy, QUIC_SVDPolicy) { - RecommendationAccuracy(); -} - -/** - * Make sure recommendations that are generated are reasonably accurate - * for regularized SVD. - */ -TEST_CASE("RecommendationAccuracyRegSVDTest", "[CFTest]") -{ - RecommendationAccuracy(); -} - -/** - * Make sure recommendations that are generated are reasonably accurate - * for batch SVD. - */ -TEST_CASE("RecommendationAccuracyBatchSVDTest", "[CFTest]") -{ - RecommendationAccuracy(); -} - -/** - * Make sure recommendations that are generated are reasonably accurate - * for NMF. - */ -TEST_CASE("RecommendationAccuracyNMFTest", "[CFTest]") -{ - RecommendationAccuracy(); -} - -/** - * Make sure recommendations that are generated are reasonably accurate - * for SVD Complete Incremental method. - */ -TEST_CASE("RecommendationAccuracySVDCompleteTest", "[CFTest]") -{ - RecommendationAccuracy(); -} - -/** - * Make sure recommendations that are generated are reasonably accurate - * for SVD Incomplete Incremental method. - */ -TEST_CASE("RecommendationAccuracySVDIncompleteTest", "[CFTest]") -{ - RecommendationAccuracy(); -} - -/** - * Make sure recommendations that are generated are reasonably accurate - * for Bias SVD method. - */ -TEST_CASE("RecommendationAccuracyBiasSVDTest", "[CFTest]") -{ - // This algorithm seems to be far less effective than others. - // We therefore allow failures on 44% of the runs. - RecommendationAccuracy(22); + RecommendationAccuracy(); } /** @@ -733,369 +540,81 @@ TEST_CASE("RecommendationAccuracyBiasSVDTest", "[CFTest]") // RecommendationAccuracy(); // } -// Make sure that Predict() is returning reasonable results for randomized SVD. -TEST_CASE("CFPredictRandSVDTest", "[CFTest]") +/** + * Make sure that Predict() is returning reasonable results for all methods. + */ +TEMPLATE_TEST_CASE("CFPredictTest", "[CFTest]", + RandomizedSVDPolicy, RegSVDPolicy, BatchSVDPolicy, NMFPolicy, + SVDCompletePolicy, SVDIncompletePolicy, BiasSVDPolicy, SVDPlusPlusPolicy, + QUIC_SVDPolicy) { - CFPredict(); -} - -// Make sure that Predict() is returning reasonable results for regularized SVD. -TEST_CASE("CFPredictRegSVDTest", "[CFTest]") -{ - CFPredict(); -} - -// Make sure that Predict() is returning reasonable results for batch SVD. -TEST_CASE("CFPredictBatchSVDTest", "[CFTest]") -{ - CFPredict(); -} - -// Make sure that Predict() is returning reasonable results for NMF. -TEST_CASE("CFPredictNMFTest", "[CFTest]") -{ - CFPredict(); + CFPredict(); } /** - * Make sure that Predict() is returning reasonable results for SVD Complete - * Incremental method. + * Compare batch Predict() and individual Predict() for all methods. */ -TEST_CASE("CFPredictSVDCompleteTest", "[CFTest]") +TEMPLATE_TEST_CASE("CFBatchPredictTest", "[CFTest]", + RandomizedSVDPolicy, RegSVDPolicy, BatchSVDPolicy, NMFPolicy, + SVDCompletePolicy, SVDIncompletePolicy, BiasSVDPolicy, SVDPlusPlusPolicy, + QUIC_SVDPolicy) { - CFPredict(); + BatchPredict(); } /** - * Make sure that Predict() is returning reasonable results for SVD Incomplete - * Incremental method. + * Make sure we can train an already-trained model and it works okay for + * some methods */ -TEST_CASE("CFPredictSVDIncompleteTest", "[CFTest]") +TEMPLATE_TEST_CASE("TrainTest_1", "[CFTest]", + RandomizedSVDPolicy, BatchSVDPolicy, NMFPolicy, SVDCompletePolicy, + SVDIncompletePolicy, QUIC_SVDPolicy) { - CFPredict(); -} - -/** - * Make sure that Predict() is returning reasonable results for Bias SVD - * method. - */ -TEST_CASE("CFPredictBiasSVDTest", "[CFTest]") -{ - CFPredict(); -} - -/** - * Make sure that Predict() is returning reasonable results for SVDPlusPlus - * method. - */ -TEST_CASE("CFPredictSVDPPTest", "[CFTest]") -{ - CFPredict(); -} - -/** - * Make sure that Predict() is returning reasonable results for Quic SVD - * method. - */ -TEST_CASE("CFPredictQSVDTest", "[CFTest]") -{ - CFPredict(); -} - -// Compare batch Predict() and individual Predict() for randomized SVD. -TEST_CASE("CFBatchPredictRandSVDTest", "[CFTest]") -{ - BatchPredict(); -} - -// Compare batch Predict() and individual Predict() for regularized SVD. -TEST_CASE("CFBatchPredictRegSVDTest", "[CFTest]") -{ - BatchPredict(); -} - -// Compare batch Predict() and individual Predict() for batch SVD. -TEST_CASE("CFBatchPredictBatchSVDTest", "[CFTest]") -{ - BatchPredict(); -} - -// Compare batch Predict() and individual Predict() for NMF. -TEST_CASE("CFBatchPredictNMFTest", "[CFTest]") -{ - BatchPredict(); -} - -// Compare batch Predict() and individual Predict() for -// SVD Complete Incremental method. -TEST_CASE("CFBatchPredictSVDCompleteTest", "[CFTest]") -{ - BatchPredict(); -} - -// Compare batch Predict() and individual Predict() for -// SVD Incomplete Incremental method. -TEST_CASE("CFBatchPredictSVDIncompleteTest", "[CFTest]") -{ - BatchPredict(); -} - -// Compare batch Predict() and individual Predict() for -// Bias SVD method. -TEST_CASE("CFBatchPredictBiasSVDTest", "[CFTest]") -{ - BatchPredict(); -} - -// Compare batch Predict() and individual Predict() for -// SVDPlusPlus method. -TEST_CASE("CFBatchPredictSVDPPTest", "[CFTest]") -{ - BatchPredict(); -} - -// Compare batch Predict() and individual Predict() for -// Quic SVD method. -TEST_CASE("CFBatchPredictQSVDTest", "[CFTest]") -{ - BatchPredict(); -} - -/** - * Make sure we can train an already-trained model and it works okay for - * randomized SVD. - */ -TEST_CASE("TrainRandSVDTest", "[CFTest]") -{ - RandomizedSVDPolicy decomposition; + TestType decomposition; Train(decomposition); } /** - * Make sure we can train an already-trained model and it works okay for - * regularized SVD. + * Make sure we can train an already-trained model and it works okay for + * some methods */ -TEST_CASE("TrainRegSVDTest", "[CFTest]") +TEMPLATE_TEST_CASE("TrainTest_2", "[CFTest]", + RegSVDPolicy, BiasSVDPolicy, SVDPlusPlusPolicy) { - RegSVDPolicy decomposition; - TrainWithCoordinateList(decomposition); -} - -/** - * Make sure we can train an already-trained model and it works okay for - * batch SVD. - */ -TEST_CASE("TrainBatchSVDTest", "[CFTest]") -{ - BatchSVDPolicy decomposition; - Train(decomposition); -} - -/** - * Make sure we can train an already-trained model and it works okay for - * NMF. - */ -TEST_CASE("TrainNMFTest", "[CFTest]") -{ - NMFPolicy decomposition; - Train(decomposition); -} - -/** - * Make sure we can train an already-trained model and it works okay for - * SVD Complete Incremental method. - */ -TEST_CASE("TrainSVDCompleteTest", "[CFTest]") -{ - SVDCompletePolicy decomposition; - Train(decomposition); -} - -/** - * Make sure we can train an already-trained model and it works okay for - * SVD Incomplete Incremental method. - */ -TEST_CASE("TrainSVDIncompleteTest", "[CFTest]") -{ - SVDIncompletePolicy decomposition; - Train(decomposition); -} - -/** - * Make sure we can train an already-trained model and it works okay for - * BiasSVD method. - */ -TEST_CASE("TrainBiasSVDTest", "[CFTest]") -{ - BiasSVDPolicy decomposition; - TrainWithCoordinateList(decomposition); -} - -/** - * Make sure we can train an already-trained model and it works okay for - * SVDPlusPlus method. - */ -TEST_CASE("TrainSVDPPTest", "[CFTest]") -{ - SVDPlusPlusPolicy decomposition; - TrainWithCoordinateList(decomposition); -} - -/** - * Make sure we can train an already-trained model and it works okay for - * Quic SVD method. - */ -TEST_CASE("TrainQSVDTest", "[CFTest]") -{ - QuicSVDPolicy decomposition; + TestType decomposition; TrainWithCoordinateList(decomposition); } /** * Make sure we can train a model after using the empty constructor when - * using randomized SVD. + * using any of the method. */ -TEST_CASE("EmptyConstructorTrainRandSVDTest", "[CFTest]") +TEMPLATE_TEST_CASE("EmptyConstructorTrainTest", "[CFTest]", + RandomizedSVDPolicy, RegSVDPolicy, BatchSVDPolicy, NMFPolicy, + SVDCompletePolicy, SVDIncompletePolicy, BiasSVDPolicy, QUIC_SVDPolicy) { - EmptyConstructorTrain(); + EmptyConstructorTrain(); } /** - * Make sure we can train a model after using the empty constructor when - * using regularized SVD. + * Ensure we can load and save the CF model using any of the method. */ -TEST_CASE("EmptyConstructorTrainRegSVDTest", "[CFTest]") +TEMPLATE_TEST_CASE("SerializationTest", "[CFTest]", + RandomizedSVDPolicy, BatchSVDPolicy, NMFPolicy, SVDCompletePolicy, + SVDIncompletePolicy, QUIC_SVDPolicy) { - EmptyConstructorTrain(); -} - -/** - * Make sure we can train a model after using the empty constructor when - * using batch SVD. - */ -TEST_CASE("EmptyConstructorTrainBatchSVDTest", "[CFTest]") -{ - EmptyConstructorTrain(); -} - -/** - * Make sure we can train a model after using the empty constructor when - * using NMF. - */ -TEST_CASE("EmptyConstructorTrainNMFTest", "[CFTest]") -{ - EmptyConstructorTrain(); -} - -/** - * Make sure we can train a model after using the empty constructor when - * using SVD Complete Incremental method. - */ -TEST_CASE("EmptyConstructorTrainSVDCompleteTest", "[CFTest]") -{ - EmptyConstructorTrain(); -} - -/** - * Make sure we can train a model after using the empty constructor when - * using SVD Incomplete Incremental method. - */ -TEST_CASE("EmptyConstructorTrainSVDIncompleteTest", "[CFTest]") -{ - EmptyConstructorTrain(); -} - -/** - * Make sure we can train a model after using the empty constructor when - * using Quic SVD method. - */ -TEST_CASE("EmptyConstructorTrainQSVDTest", "[CFTest]") -{ - EmptyConstructorTrain(); -} - -/** - * Ensure we can load and save the CF model using randomized SVD policy. - */ -TEST_CASE("SerializationRandSVDTest", "[CFTest]") -{ - Serialization(); -} - -/** - * Ensure we can load and save the CF model using batch SVD policy. - */ -TEST_CASE("SerializationBatchSVDTest", "[CFTest]") -{ - Serialization(); -} - -/** - * Ensure we can load and save the CF model using NMF policy. - */ -TEST_CASE("SerializationNMFTest", "[CFTest]") -{ - Serialization(); -} - -/** - * Ensure we can load and save the CF model using SVD Complete Incremental. - */ -TEST_CASE("SerializationSVDCompleteTest", "[CFTest]") -{ - Serialization(); -} - -/** - * Ensure we can load and save the CF model using SVD Incomplete Incremental. - */ -TEST_CASE("SerializationSVDIncompleteTest", "[CFTest]") -{ - Serialization(); -} - -/** - * Ensure we can load and save the CF model using Quic SVD Policy. - */ -TEST_CASE("SerializationQSVDTest", "[CFTest]") -{ - Serialization(); + Serialization(); } /** * Make sure that Predict() is returning reasonable results for NMF and - * OverallMeanNormalization. + * all types of Normalization except default. */ -TEST_CASE("CFPredictOverallMeanNormalization", "[CFTest]") +TEMPLATE_TEST_CASE("CFPredictNormalization", "[CFTest]", + OverallMeanNormalization, UserMeanNormalization, ItemMeanNormalization, + ZScoreNormalization) { - CFPredict(2.0); -} - -/** - * Make sure that Predict() is returning reasonable results for NMF and - * UserMeanNormalization. - */ -TEST_CASE("CFPredictUserMeanNormalization", "[CFTest]") -{ - CFPredict(2.0); -} - -/** - * Make sure that Predict() is returning reasonable results for NMF and - * ItemMeanNormalization. - */ -TEST_CASE("CFPredictItemMeanNormalization", "[CFTest]") -{ - CFPredict(2.0); -} - -/** - * Make sure that Predict() is returning reasonable results for NMF and - * ZScoreNormalization. - */ -TEST_CASE("CFPredictZScoreNormalization", "[CFTest]") -{ - CFPredict(2.0); + CFPredict(2.0); } /** @@ -1122,38 +641,13 @@ TEST_CASE("CFPredictNoNormalization", "[CFTest]") /** * Make sure recommendations that are generated are reasonably accurate - * for OverallMeanNormalization. + * for all types of Normalization except default. */ -TEST_CASE("RecommendationAccuracyOverallMeanNormalizationTest", "[CFTest]") +TEMPLATE_TEST_CASE("RecommendationAccuracyNormalizationTest", "[CFTest]", + OverallMeanNormalization, UserMeanNormalization, ItemMeanNormalization, + ZScoreNormalization) { - RecommendationAccuracy(); -} - -/** - * Make sure recommendations that are generated are reasonably accurate - * for UserMeanNormalization. - */ -TEST_CASE("RecommendationAccuracyUserMeanNormalizationTest", "[CFTest]") -{ - RecommendationAccuracy(); -} - -/** - * Make sure recommendations that are generated are reasonably accurate - * for ItemMeanNormalization. - */ -TEST_CASE("RecommendationAccuracyItemMeanNormalizationTest", "[CFTest]") -{ - RecommendationAccuracy(); -} - -/** - * Make sure recommendations that are generated are reasonably accurate - * for ZScoreNormalization. - */ -TEST_CASE("RecommendationAccuracyZScoreNormalizationTest", "[CFTest]") -{ - RecommendationAccuracy(); + RecommendationAccuracy(); } /** @@ -1170,35 +664,14 @@ TEST_CASE("RecommendationAccuracyCombinedNormalizationTest", "[CFTest]") } /** - * Ensure we can load and save the CF model using OverallMeanNormalization. + * Ensure we can load and save the CF model using any type of Normalization + * except default. */ -TEST_CASE("SerializationOverallMeanNormalizationTest", "[CFTest]") +TEMPLATE_TEST_CASE("SerializationNormalizationTest", "[CFTest]", + OverallMeanNormalization, UserMeanNormalization, ItemMeanNormalization, + ZScoreNormalization) { - Serialization(); -} - -/** - * Ensure we can load and save the CF model using UserMeanNormalization. - */ -TEST_CASE("SerializationUserMeanNormalizationTest", "[CFTest]") -{ - Serialization(); -} - -/** - * Ensure we can load and save the CF model using ItemMeanNormalization. - */ -TEST_CASE("SerializationItemMeanNormalizationTest", "[CFTest]") -{ - Serialization(); -} - -/** - * Ensure we can load and save the CF model using ZScoreMeanNormalization. - */ -TEST_CASE("SerializationZScoreNormalizationTest", "[CFTest]") -{ - Serialization(); + Serialization(); } /** @@ -1214,54 +687,24 @@ TEST_CASE("SerializationCombinedNormalizationTest", "[CFTest]") } /** - * Make sure that Predict() is returning reasonable results for - * EuclideanSearch. + * Make sure that Predict() is returning reasonable results for all search + * except default. */ -TEST_CASE("CFPredictEuclideanSearch", "[CFTest]") +TEMPLATE_TEST_CASE("CFPredictSearch", "[CFTest]", + EuclideanSearch, CosineSearch, PearsonSearch) { - CFPredict(2.0); + CFPredict(2.0); } /** * Make sure that Predict() is returning reasonable results for - * CosineSearch. + * some Interpolations. */ -TEST_CASE("CFPredictCosineSearch", "[CFTest]") +TEMPLATE_TEST_CASE("CFPredictAverageInterpolation", "[CFTest]", + AverageInterpolation, SimilarityInterpolation) { - CFPredict(2.0); -} - -/** - * Make sure that Predict() is returning reasonable results for - * PearsonSearch. - */ -TEST_CASE("CFPredictPearsonSearch", "[CFTest]") -{ - CFPredict(2.0); -} - -/** - * Make sure that Predict() is returning reasonable results for - * AverageInterpolation. - */ -TEST_CASE("CFPredictAverageInterpolation", "[CFTest]") -{ - CFPredict(2.0); -} - -/** - * Make sure that Predict() is returning reasonable results for - * SimilarityInterpolation. - */ -TEST_CASE("CFPredictSimilarityInterpolation", "[CFTest]") -{ - CFPredict(2.0); + CFPredict(2.0); } /** From caf5543a94322f118365aaa18f07a7c71bbb7b0d Mon Sep 17 00:00:00 2001 From: AdarshSantoria Date: Tue, 14 Feb 2023 02:28:17 +0530 Subject: [PATCH 50/88] fix bksvd method --- .../cf/decomposition_policies/block_krylov_svd_method.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp b/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp index d924ba8ec6..7b721e8ff4 100644 --- a/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp +++ b/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp @@ -76,6 +76,9 @@ class BlockKrylovSVDPolicy // Do singular value decomposition using the block krylov SVD algorithm. RandomizedBlockKrylovSVD blockkrylovsvd; blockkrylovsvd.Apply(data, w, sigma, h, rank); + + // Sigma matrix is multiplied to w. + w = w * sigma; // Take transpose of the matrix h as required by CF class. h = arma::trans(h); From e6616bd33367d8ecbfbcf385535ce190bc386912 Mon Sep 17 00:00:00 2001 From: AdarshSantoria Date: Tue, 14 Feb 2023 02:29:12 +0530 Subject: [PATCH 51/88] fix bksvd method --- .../cf/decomposition_policies/block_krylov_svd_method.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp b/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp index 7b721e8ff4..fdc367d54f 100644 --- a/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp +++ b/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp @@ -76,9 +76,9 @@ class BlockKrylovSVDPolicy // Do singular value decomposition using the block krylov SVD algorithm. RandomizedBlockKrylovSVD blockkrylovsvd; blockkrylovsvd.Apply(data, w, sigma, h, rank); - + // Sigma matrix is multiplied to w. - w = w * sigma; + w = w * arma::diagmat(sigma); // Take transpose of the matrix h as required by CF class. h = arma::trans(h); From 47e0b039e10d9e5d510c68d80b9d964e759d64d1 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Tue, 14 Feb 2023 21:41:53 +0530 Subject: [PATCH 52/88] Update src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp Co-authored-by: Ryan Curtin --- .../cf/decomposition_policies/block_krylov_svd_method.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp b/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp index fdc367d54f..f49046a42e 100644 --- a/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp +++ b/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp @@ -20,8 +20,8 @@ namespace mlpack { /** - * Implementation of the BLOCK-KRYLOV-SVD policy to act as a wrapper when - * accessing BLOCK KRYLOV SVD from within CFType. + * Implementation of the Block Krylov SVD policy to act as a wrapper when + * using Block Krylov SVD for the decomposition type of CF. * * An example of how to use BlockKrylovSVDPolicy in CF is shown below: * From 41bc55b37ebaff06135c75663976e58e338d7ec4 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Tue, 14 Feb 2023 21:42:04 +0530 Subject: [PATCH 53/88] Update src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp Co-authored-by: Ryan Curtin --- .../cf/decomposition_policies/block_krylov_svd_method.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp b/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp index f49046a42e..61b3d3f22d 100644 --- a/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp +++ b/src/mlpack/methods/cf/decomposition_policies/block_krylov_svd_method.hpp @@ -41,7 +41,7 @@ class BlockKrylovSVDPolicy { public: /** - * Use block krylov SVD method to perform collaborative filtering + * Create block krylov SVD object to use for collaborative filtering. */ BlockKrylovSVDPolicy() { From 6066623f5ea8832126a1c51359aad80125b66931 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Tue, 14 Feb 2023 21:53:44 +0530 Subject: [PATCH 54/88] Update src/mlpack/methods/quic_svd/quic_svd_impl.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/quic_svd/quic_svd_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/quic_svd/quic_svd_impl.hpp b/src/mlpack/methods/quic_svd/quic_svd_impl.hpp index d44c88c1b9..9622e55015 100644 --- a/src/mlpack/methods/quic_svd/quic_svd_impl.hpp +++ b/src/mlpack/methods/quic_svd/quic_svd_impl.hpp @@ -59,7 +59,7 @@ inline void QUIC_SVD::Apply( // Use the ExtractSVD algorithm mentioned in the paper to extract the SVD of // the original dataset in the obtained subspace. - ExtractSVD(dataset,u, v, sigma); + ExtractSVD(dataset, u, v, sigma); } inline void QUIC_SVD::ExtractSVD(const arma::mat& dataset, From 70539c8b28da7396fffc686c6ec17a147ecb57a1 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Tue, 14 Feb 2023 21:53:53 +0530 Subject: [PATCH 55/88] Update src/mlpack/tests/cf_test.cpp Co-authored-by: Ryan Curtin --- src/mlpack/tests/cf_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index db6232c81e..d8b527cf17 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -701,7 +701,7 @@ TEMPLATE_TEST_CASE("CFPredictSearch", "[CFTest]", * some Interpolations. */ TEMPLATE_TEST_CASE("CFPredictAverageInterpolation", "[CFTest]", - AverageInterpolation, SimilarityInterpolation) + AverageInterpolation, SimilarityInterpolation) { CFPredict(2.0); From bcacf4efdd6b2edcffa683af553ae3033edf683a Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Tue, 14 Feb 2023 21:54:05 +0530 Subject: [PATCH 56/88] Update src/mlpack/tests/cf_test.cpp Co-authored-by: Ryan Curtin --- src/mlpack/tests/cf_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index d8b527cf17..ffda552cec 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -499,9 +499,9 @@ void Serialization() * set for all methods. */ TEMPLATE_TEST_CASE("CFGetRecommendationsAllUsersTest", "[CFTest]", - RandomizedSVDPolicy, RegSVDPolicy, BatchSVDPolicy, NMFPolicy, - SVDCompletePolicy, SVDIncompletePolicy, BiasSVDPolicy, SVDPlusPlusPolicy, - QUIC_SVDPolicy) + RandomizedSVDPolicy, RegSVDPolicy, BatchSVDPolicy, NMFPolicy, + SVDCompletePolicy, SVDIncompletePolicy, BiasSVDPolicy, SVDPlusPlusPolicy, + QUIC_SVDPolicy) { GetRecommendationsAllUsers(); } From 32eaf10b3d9207545ba1c78a7bf8cd6844a24c37 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Tue, 14 Feb 2023 21:54:15 +0530 Subject: [PATCH 57/88] Update src/mlpack/tests/cf_test.cpp Co-authored-by: Ryan Curtin --- src/mlpack/tests/cf_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index ffda552cec..37a4ce4b71 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -523,8 +523,8 @@ TEMPLATE_TEST_CASE("CFGetRecommendationsQueriedUsersTest", "[CFTest]", * for all methods except SVDPlusPlus method. */ TEMPLATE_TEST_CASE("RecommendationAccuracyTest", "[CFTest]", - RandomizedSVDPolicy, RegSVDPolicy, BatchSVDPolicy, NMFPolicy, - SVDCompletePolicy, SVDIncompletePolicy, BiasSVDPolicy, QUIC_SVDPolicy) + RandomizedSVDPolicy, RegSVDPolicy, BatchSVDPolicy, NMFPolicy, + SVDCompletePolicy, SVDIncompletePolicy, BiasSVDPolicy, QUIC_SVDPolicy) { RecommendationAccuracy(); } From 3f9367c26df140b556b6256add87eb624ed6df9a Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Tue, 14 Feb 2023 21:54:23 +0530 Subject: [PATCH 58/88] Update src/mlpack/tests/cf_test.cpp Co-authored-by: Ryan Curtin --- src/mlpack/tests/cf_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index 37a4ce4b71..9e0e69ff30 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -544,9 +544,9 @@ TEMPLATE_TEST_CASE("RecommendationAccuracyTest", "[CFTest]", * Make sure that Predict() is returning reasonable results for all methods. */ TEMPLATE_TEST_CASE("CFPredictTest", "[CFTest]", - RandomizedSVDPolicy, RegSVDPolicy, BatchSVDPolicy, NMFPolicy, - SVDCompletePolicy, SVDIncompletePolicy, BiasSVDPolicy, SVDPlusPlusPolicy, - QUIC_SVDPolicy) + RandomizedSVDPolicy, RegSVDPolicy, BatchSVDPolicy, NMFPolicy, + SVDCompletePolicy, SVDIncompletePolicy, BiasSVDPolicy, SVDPlusPlusPolicy, + QUIC_SVDPolicy) { CFPredict(); } From 84e4ec7367b3480f01c27ff7ce7cf329801ad4cb Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Tue, 14 Feb 2023 21:54:31 +0530 Subject: [PATCH 59/88] Update src/mlpack/tests/cf_test.cpp Co-authored-by: Ryan Curtin --- src/mlpack/tests/cf_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index 9e0e69ff30..8a42c28d72 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -555,9 +555,9 @@ TEMPLATE_TEST_CASE("CFPredictTest", "[CFTest]", * Compare batch Predict() and individual Predict() for all methods. */ TEMPLATE_TEST_CASE("CFBatchPredictTest", "[CFTest]", - RandomizedSVDPolicy, RegSVDPolicy, BatchSVDPolicy, NMFPolicy, - SVDCompletePolicy, SVDIncompletePolicy, BiasSVDPolicy, SVDPlusPlusPolicy, - QUIC_SVDPolicy) + RandomizedSVDPolicy, RegSVDPolicy, BatchSVDPolicy, NMFPolicy, + SVDCompletePolicy, SVDIncompletePolicy, BiasSVDPolicy, SVDPlusPlusPolicy, + QUIC_SVDPolicy) { BatchPredict(); } From fd58849f5eb8c02df0fb1bfea9f336bdc8748137 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Tue, 14 Feb 2023 21:54:40 +0530 Subject: [PATCH 60/88] Update src/mlpack/tests/cf_test.cpp Co-authored-by: Ryan Curtin --- src/mlpack/tests/cf_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index 8a42c28d72..ceb3d71627 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -691,7 +691,7 @@ TEST_CASE("SerializationCombinedNormalizationTest", "[CFTest]") * except default. */ TEMPLATE_TEST_CASE("CFPredictSearch", "[CFTest]", - EuclideanSearch, CosineSearch, PearsonSearch) + EuclideanSearch, CosineSearch, PearsonSearch) { CFPredict(2.0); } From 30f57637c5c17c418ecafba5e757fdb663fe2042 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Tue, 14 Feb 2023 21:54:49 +0530 Subject: [PATCH 61/88] Update src/mlpack/tests/cf_test.cpp Co-authored-by: Ryan Curtin --- src/mlpack/tests/cf_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index ceb3d71627..b13029f611 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -704,7 +704,7 @@ TEMPLATE_TEST_CASE("CFPredictAverageInterpolation", "[CFTest]", AverageInterpolation, SimilarityInterpolation) { CFPredict(2.0); + TestType>(2.0); } /** From ddd491a76a6e68387291f31fab4236e153b4a8a6 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Tue, 14 Feb 2023 21:55:04 +0530 Subject: [PATCH 62/88] Update src/mlpack/tests/cf_test.cpp Co-authored-by: Ryan Curtin --- src/mlpack/tests/cf_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index b13029f611..9e9b919fef 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -567,8 +567,8 @@ TEMPLATE_TEST_CASE("CFBatchPredictTest", "[CFTest]", * some methods */ TEMPLATE_TEST_CASE("TrainTest_1", "[CFTest]", - RandomizedSVDPolicy, BatchSVDPolicy, NMFPolicy, SVDCompletePolicy, - SVDIncompletePolicy, QUIC_SVDPolicy) + RandomizedSVDPolicy, BatchSVDPolicy, NMFPolicy, SVDCompletePolicy, + SVDIncompletePolicy, QUIC_SVDPolicy) { TestType decomposition; Train(decomposition); From b760dd72edace5058171ebdbd63efe40282f6845 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Tue, 14 Feb 2023 21:58:50 +0530 Subject: [PATCH 63/88] Revert change --- src/mlpack/methods/quic_svd/quic_svd_impl.hpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/mlpack/methods/quic_svd/quic_svd_impl.hpp b/src/mlpack/methods/quic_svd/quic_svd_impl.hpp index 9622e55015..eb010c6e16 100644 --- a/src/mlpack/methods/quic_svd/quic_svd_impl.hpp +++ b/src/mlpack/methods/quic_svd/quic_svd_impl.hpp @@ -82,9 +82,6 @@ inline void QUIC_SVD::ExtractSVD(const arma::mat& dataset, arma::vec sigmaBar; arma::svd(uBar, sigmaBar, vBar, projectedMatSquared); - // vBar was tarnsposed of vBar - vBar = arma::trans(vBar); - // Calculate the approximate SVD of the original matrix, using the SVD of the // squared projected matrix. v = basis * vBar; From 232ed1cfee11726d7536d9b64b40e95aeb52bba4 Mon Sep 17 00:00:00 2001 From: AdarshSantoria Date: Wed, 15 Feb 2023 00:07:26 +0530 Subject: [PATCH 64/88] Fix PReLU --- src/mlpack/methods/ann/layer/layer_types.hpp | 1 + .../not_adapted/parametric_relu_impl.hpp | 88 ----------- .../{not_adapted => }/parametric_relu.hpp | 48 +++--- .../ann/layer/parametric_relu_impl.hpp | 140 ++++++++++++++++++ .../tests/ann/layer/parametric_relu.cpp | 86 +++++++++++ src/mlpack/tests/ann/layer_test.cpp | 1 + 6 files changed, 257 insertions(+), 107 deletions(-) delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/parametric_relu_impl.hpp rename src/mlpack/methods/ann/layer/{not_adapted => }/parametric_relu.hpp (72%) create mode 100644 src/mlpack/methods/ann/layer/parametric_relu_impl.hpp create mode 100644 src/mlpack/tests/ann/layer/parametric_relu.cpp diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index a0984f2eee..779cdc2716 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include diff --git a/src/mlpack/methods/ann/layer/not_adapted/parametric_relu_impl.hpp b/src/mlpack/methods/ann/layer/not_adapted/parametric_relu_impl.hpp deleted file mode 100644 index bab4f9563f..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/parametric_relu_impl.hpp +++ /dev/null @@ -1,88 +0,0 @@ -/** - * @file methods/ann/layer/parametric_relu_impl.hpp - * @author Prasanna Patil - * - * Definition of PReLU layer first introduced in the, - * Kaiming He, Xiangyu Zhang, Shaoqing, Ren Jian Sun, - * "Delving Deep into Rectifiers: - * Surpassing Human-Level Performance on ImageNet Classification", 2014 - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_PRELU_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_PRELU_IMPL_HPP - -// In case it hasn't yet been included. -#include "parametric_relu.hpp" - -namespace mlpack { - -template -PReLUType::PReLUType( - const double userAlpha) : userAlpha(userAlpha) -{ - alpha.set_size(WeightSize(), 1); - alpha(0) = userAlpha; -} - -template -void PReLUType::SetWeights( - typename OutputType::elem_type* weightsPtr) -{ - alpha = arma::mat(weightsPtr, 1, 1, false, false); - - //! Set value of alpha to the one given by user. - // TODO: this doesn't even make any sense. is it trainable or not? - // why is there userAlpha? is that for initialization only? - alpha(0) = userAlpha; -} - -template -void PReLUType::Forward( - const InputType& input, OutputType& output) -{ - // TODO: use transform()? - output = input; - arma::uvec negative = arma::find(input < 0); - output(negative) = input(negative) * alpha(0); -} - -template -void PReLUType::Backward( - const InputType& input, const OutputType& gy, OutputType& g) -{ - OutputType derivative; - derivative.set_size(arma::size(input)); - for (size_t i = 0; i < input.n_elem; ++i) - derivative(i) = (input(i) >= 0) ? 1 : alpha(0); - - g = gy % derivative; -} - -template -void PReLUType::Gradient( - const InputType& input, - const OutputType& error, - OutputType& gradient) -{ - OutputType zeros = arma::zeros(input.n_rows, input.n_cols); - gradient(0) = arma::accu(error % arma::min(zeros, input)) / input.n_cols; -} - -template -template -void PReLUType::serialize( - Archive& ar, - const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); - - ar(CEREAL_NVP(alpha)); -} - -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/parametric_relu.hpp b/src/mlpack/methods/ann/layer/parametric_relu.hpp similarity index 72% rename from src/mlpack/methods/ann/layer/not_adapted/parametric_relu.hpp rename to src/mlpack/methods/ann/layer/parametric_relu.hpp index 948d0722c0..6edd851059 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/parametric_relu.hpp +++ b/src/mlpack/methods/ann/layer/parametric_relu.hpp @@ -12,8 +12,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_METHODS_ANN_LAYER_PReLU_HPP -#define MLPACK_METHODS_ANN_LAYER_PReLU_HPP +#ifndef MLPACK_METHODS_ANN_LAYER_PRELU_HPP +#define MLPACK_METHODS_ANN_LAYER_PRELU_HPP #include @@ -34,14 +34,12 @@ namespace mlpack { * \right. * @f} * - * @tparam InputType The type of the layer's inputs. The layer automatically - * cast inputs to this type (Default: arma::mat). - * @tparam OutputType The type of the computation which also causes the output - * to also be in this type. The type also allows the computation and weight - * type to differ from the input type (Default: arma::mat). + * @tparam MatType Matrix representation to accept as input and allows the + * computation and weight type to differ from the input type + * (Default: arma::mat). */ -template -class PReLUType : public Layer +template +class PReLUType : public Layer { public: /** @@ -57,8 +55,20 @@ class PReLUType : public Layer //! Clone the PReLUType object. This handles polymorphism correctly. PReLUType* Clone() const { return new PReLUType(*this); } + // Virtual destructor. + virtual ~PReLUType() { } + + //! Copy the given PReLUType. + PReLUType(const PReLUType& other); + //! Take ownership of the given PReLUType. + PReLUType(PReLUType&& other); + //! Copy the given PReLUType. + PReLUType& operator=(const PReLUType& other); + //! Take ownership of the given PReLUType. + PReLUType& operator=(PReLUType&& other); + //! Reset the layer parameter. - void SetWeights(typename OutputType::elem_type* weightsPtr); + void SetWeights(typename MatType::elem_type* weightsPtr); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -67,7 +77,7 @@ class PReLUType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + void Forward(const MatType& input, MatType& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -78,7 +88,7 @@ class PReLUType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, const OutputType& gy, OutputType& g); + void Backward(const MatType& input, const MatType& gy, MatType& g); /** * Calculate the gradient using the output delta and the input activation. @@ -87,14 +97,14 @@ class PReLUType : public Layer * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const InputType& input, - const OutputType& error, - OutputType& gradient); + void Gradient(const MatType& input, + const MatType& error, + MatType& gradient); //! Get the parameters. - OutputType const& Parameters() const { return alpha; } + MatType const& Parameters() const { return alpha; } //! Modify the parameters. - OutputType& Parameters() { return alpha; } + MatType& Parameters() { return alpha; } //! Get the non zero gradient. double const& Alpha() const { return alpha(0); } @@ -112,7 +122,7 @@ class PReLUType : public Layer private: //! Leakyness Parameter object. - OutputType alpha; + MatType alpha; //! Leakyness Parameter given by user in the range 0 < alpha < 1. double userAlpha; @@ -121,7 +131,7 @@ class PReLUType : public Layer // Convenience typedefs. // Standard PReLU layer. -typedef PReLUType PReLU; +typedef PReLUType PReLU; } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp new file mode 100644 index 0000000000..5a8af8da14 --- /dev/null +++ b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp @@ -0,0 +1,140 @@ +/** + * @file methods/ann/layer/parametric_relu_impl.hpp + * @author Prasanna Patil + * + * Definition of PReLU layer first introduced in the, + * Kaiming He, Xiangyu Zhang, Shaoqing, Ren Jian Sun, + * "Delving Deep into Rectifiers: + * Surpassing Human-Level Performance on ImageNet Classification", 2014 + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_PRELU_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_PRELU_IMPL_HPP + +// In case it hasn't yet been included. +#include "parametric_relu.hpp" + +namespace mlpack { + +template +PReLUType::PReLUType(const double userAlpha) : + Layer(), + userAlpha(userAlpha) +{ + alpha.set_size(WeightSize(), 1); + alpha(0) = userAlpha; +} + +template +PReLUType::PReLUType( + const PReLUType& other) : + Layer(other), + userAlpha(other.userAlpha) +{ + alpha.set_size(WeightSize(), 1); + alpha(0) = userAlpha; +} + +template +PReLUType::PReLUType( + PReLUType&& other) : + Layer(std::move(other)), + userAlpha(std::move(other.userAlpha)) +{ + alpha.set_size(WeightSize(), 1); + alpha(0) = userAlpha; +} + +template +PReLUType& +PReLUType::operator=(const PReLUType& other) +{ + if (&other != this) + { + Layer::operator=(other); + userAlpha = other.userAlpha; + alpha = other.alpha; + } + + return *this; +} + +template +PReLUType& +PReLUType::operator=(PReLUType&& other) +{ + if (&other != this) + { + Layer::operator=(std::move(other)); + userAlpha = std::move(other.userAlpha); + alpha = std::move(other.alpha); + } + + return *this; +} + +template +void PReLUType::SetWeights( + typename MatType::elem_type* weightsPtr) +{ + alpha = arma::mat(weightsPtr, 1, 1, false, false); + + //! Set value of alpha to the one given by user. + // TODO: this doesn't even make any sense. is it trainable or not? + // why is there userAlpha? is that for initialization only? + alpha(0) = userAlpha; +} + +template +void PReLUType::Forward( + const MatType& input, MatType& output) +{ + // TODO: use transform()? + output = input; + #pragma omp for + for (size_t i = 0; i < input.n_elem; ++i) + output(i) *= (input(i) >= 0) ? 1 : alpha(0); +} + +template +void PReLUType::Backward( + const MatType& input, const MatType& gy, MatType& g) +{ + MatType derivative; + derivative.set_size(arma::size(input)); + #pragma omp for + for (size_t i = 0; i < input.n_elem; ++i) + derivative(i) = (input(i) >= 0) ? 1 : alpha(0); + + g = gy % derivative; +} + +template +void PReLUType::Gradient( + const MatType& input, + const MatType& error, + MatType& gradient) +{ + MatType zeros = arma::zeros(input.n_rows, input.n_cols); + gradient.set_size(WeightSize(), 1); + gradient(0) = arma::accu(error % arma::min(zeros, input)) / input.n_cols; +} + +template +template +void PReLUType::serialize( + Archive& ar, + const uint32_t /* version */) +{ + ar(cereal::base_class>(this)); + + ar(CEREAL_NVP(alpha)); +} + +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/ann/layer/parametric_relu.cpp b/src/mlpack/tests/ann/layer/parametric_relu.cpp new file mode 100644 index 0000000000..e6fe060ea4 --- /dev/null +++ b/src/mlpack/tests/ann/layer/parametric_relu.cpp @@ -0,0 +1,86 @@ +/** + * @file tests/ann/layer/parametric_relu.cpp + * @author Adarsh Santoria + * + * Tests the parametric relu layer modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include +#include + +#include "../../test_catch_tools.hpp" +#include "../../catch.hpp" +#include "../../serialization.hpp" +#include "../ann_test_tools.hpp" + +using namespace mlpack; + +/** + * PReLU FORWARD Test. + */ +TEST_CASE("PReLUFORWARDTest", "[ANNLayerTest]") +{ + arma::mat input = {{0.5, 1.2, 3.1}, + {-2.2, -1.5, 0.8}, + {5.5, -4.7, 2.1}, + {0.2, 0.1, -0.5}}; + PReLU model(0.01); + arma::mat predOutput; + model.Forward(input, predOutput); + arma::mat actualOutput = {{0.5, 1.2, 3.1}, + {-0.022, -0.015, 0.8}, + {5.5, -0.047, 2.1}, + {0.2, 0.1, -0.005}}; + REQUIRE(arma::accu(arma::abs(actualOutput - predOutput)) == + Approx(0.0).margin(1e-4)); +} + +/** + * PReLU BACKWARD Test. + */ +TEST_CASE("PReLUBACKWARDTest", "[ANNLayerTest]") +{ + arma::mat input = {{0.5, 1.2, 3.1}, + {-2.2, -1.5, 0.8}, + {5.5, -4.7, 2.1}, + {0.2, 0.1, -0.5}}; + PReLU model(0.01); + arma::mat gy = {{0.2, -0.5, 0.8}, + {1.5, -0.6, 0.1}, + {-0.3, 0.2, -0.5}, + {0.1, -0.1, 0.3}}; + arma::mat predG; + model.Backward(input, gy, predG); + arma::mat actualG = {{0.2, -0.5, 0.8}, + {0.015, -0.006, 0.1}, + {-0.3, 0.002, -0.5}, + {0.1, -0.1, 0.0030}}; + + REQUIRE(arma::accu(arma::abs(actualG - predG)) == + Approx(0.0).margin(1e-4)); +} + +/** + * PReLU GRADIENT Test. + */ +TEST_CASE("PReLUGRADIENTTest", "[ANNLayerTest]") +{ + arma::mat input = {{0.5, 1.2, 3.1}, + {-2.2, -1.5, 0.8}, + {5.5, -4.7, 2.1}, + {0.2, 0.1, -0.5}}; + PReLU model(0.01); + arma::mat error = {{0.2, -0.5, 0.8}, + {-0.015, -0.006, 0.001}, + {-0.3, 0.002, -0.005}, + {0.1, -0.1, 0.0035}}; + arma::mat predGradient; + model.Gradient(input, error, predGradient); + + REQUIRE(0.0103 - arma::accu(predGradient) == + Approx(0.0).margin(1e-4)); +} \ No newline at end of file diff --git a/src/mlpack/tests/ann/layer_test.cpp b/src/mlpack/tests/ann/layer_test.cpp index e22bf8573f..2acbfb053e 100644 --- a/src/mlpack/tests/ann/layer_test.cpp +++ b/src/mlpack/tests/ann/layer_test.cpp @@ -31,4 +31,5 @@ #include "layer/max_pooling.cpp" #include "layer/mean_pooling.cpp" #include "layer/padding.cpp" +#include "layer/parametric_relu.cpp" #include "layer/softmax.cpp" From 81033aa3265b960609a22ae3fcfc5b476f413bf9 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Wed, 15 Feb 2023 00:55:47 +0530 Subject: [PATCH 65/88] Update src/mlpack/tests/cf_test.cpp Co-authored-by: Ryan Curtin --- src/mlpack/tests/cf_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index 9e9b919fef..afa1e73e97 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -644,8 +644,8 @@ TEST_CASE("CFPredictNoNormalization", "[CFTest]") * for all types of Normalization except default. */ TEMPLATE_TEST_CASE("RecommendationAccuracyNormalizationTest", "[CFTest]", - OverallMeanNormalization, UserMeanNormalization, ItemMeanNormalization, - ZScoreNormalization) + OverallMeanNormalization, UserMeanNormalization, ItemMeanNormalization, + ZScoreNormalization) { RecommendationAccuracy(); } From ad5e1c46d58e224ebf9895c321038cd388142f8a Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Wed, 15 Feb 2023 00:56:13 +0530 Subject: [PATCH 66/88] Update src/mlpack/tests/cf_test.cpp Co-authored-by: Ryan Curtin --- src/mlpack/tests/cf_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index afa1e73e97..5db3b1a6bf 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -579,7 +579,7 @@ TEMPLATE_TEST_CASE("TrainTest_1", "[CFTest]", * some methods */ TEMPLATE_TEST_CASE("TrainTest_2", "[CFTest]", - RegSVDPolicy, BiasSVDPolicy, SVDPlusPlusPolicy) + RegSVDPolicy, BiasSVDPolicy, SVDPlusPlusPolicy) { TestType decomposition; TrainWithCoordinateList(decomposition); From 35a82eceb8171935ef04d8e1ff486f0825e47511 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Wed, 15 Feb 2023 00:56:32 +0530 Subject: [PATCH 67/88] Update src/mlpack/tests/cf_test.cpp Co-authored-by: Ryan Curtin --- src/mlpack/tests/cf_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index 5db3b1a6bf..195f5a029d 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -668,8 +668,8 @@ TEST_CASE("RecommendationAccuracyCombinedNormalizationTest", "[CFTest]") * except default. */ TEMPLATE_TEST_CASE("SerializationNormalizationTest", "[CFTest]", - OverallMeanNormalization, UserMeanNormalization, ItemMeanNormalization, - ZScoreNormalization) + OverallMeanNormalization, UserMeanNormalization, ItemMeanNormalization, + ZScoreNormalization) { Serialization(); } From 089863f4687071ff320d8ab43d28eb70f83b91d1 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Wed, 15 Feb 2023 00:56:48 +0530 Subject: [PATCH 68/88] Update src/mlpack/tests/cf_test.cpp Co-authored-by: Ryan Curtin --- src/mlpack/tests/cf_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index 195f5a029d..387cf45c3d 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -600,8 +600,8 @@ TEMPLATE_TEST_CASE("EmptyConstructorTrainTest", "[CFTest]", * Ensure we can load and save the CF model using any of the method. */ TEMPLATE_TEST_CASE("SerializationTest", "[CFTest]", - RandomizedSVDPolicy, BatchSVDPolicy, NMFPolicy, SVDCompletePolicy, - SVDIncompletePolicy, QUIC_SVDPolicy) + RandomizedSVDPolicy, BatchSVDPolicy, NMFPolicy, SVDCompletePolicy, + SVDIncompletePolicy, QUIC_SVDPolicy) { Serialization(); } From b8190d18249e105ca146060b3083ca0f057e11e6 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Wed, 15 Feb 2023 00:57:07 +0530 Subject: [PATCH 69/88] Update src/mlpack/tests/cf_test.cpp Co-authored-by: Ryan Curtin --- src/mlpack/tests/cf_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index 387cf45c3d..516e76619d 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -611,8 +611,8 @@ TEMPLATE_TEST_CASE("SerializationTest", "[CFTest]", * all types of Normalization except default. */ TEMPLATE_TEST_CASE("CFPredictNormalization", "[CFTest]", - OverallMeanNormalization, UserMeanNormalization, ItemMeanNormalization, - ZScoreNormalization) + OverallMeanNormalization, UserMeanNormalization, ItemMeanNormalization, + ZScoreNormalization) { CFPredict(2.0); } From ab7277d2a22226d754413e4b4c27ae325b7e31d6 Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Wed, 15 Feb 2023 06:19:05 -0600 Subject: [PATCH 70/88] [R] Also remove cmake copying step for configure script --- src/mlpack/bindings/R/CMakeLists.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/mlpack/bindings/R/CMakeLists.txt b/src/mlpack/bindings/R/CMakeLists.txt index b8ff0ca988..873d01ca0c 100644 --- a/src/mlpack/bindings/R/CMakeLists.txt +++ b/src/mlpack/bindings/R/CMakeLists.txt @@ -289,11 +289,6 @@ if (BUILD_R_BINDINGS) DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/mlpack/") - file(COPY - "${CMAKE_CURRENT_SOURCE_DIR}/mlpack/configure" - DESTINATION - "${CMAKE_CURRENT_BINARY_DIR}/mlpack/") - # Do the actual build. add_custom_target(r_build ALL) From b61216132f7110e55e99ea57914181faeaa17ce0 Mon Sep 17 00:00:00 2001 From: Rodo Date: Thu, 16 Feb 2023 15:15:28 +1000 Subject: [PATCH 71/88] Fix function name (#3421) --- doc/tutorials/image.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/image.md b/doc/tutorials/image.md index 01169e7afb..4303a9f9f7 100644 --- a/doc/tutorials/image.md +++ b/doc/tutorials/image.md @@ -103,7 +103,7 @@ bool Load(const std::vector& files, ```c++ data::ImageInfo info; std::vector> files{"test_image1.bmp","test_image2.bmp"}; -data::load(files, matrix, info, false, true); +data::Load(files, matrix, info, false, true); ``` ## Saving From fc0afb80c5b4e469b6b613eeae159c4e059fed9d Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Fri, 17 Feb 2023 10:35:21 +0530 Subject: [PATCH 72/88] add new line at bottom --- src/mlpack/tests/ann/layer/parametric_relu.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/ann/layer/parametric_relu.cpp b/src/mlpack/tests/ann/layer/parametric_relu.cpp index e6fe060ea4..cc4680a398 100644 --- a/src/mlpack/tests/ann/layer/parametric_relu.cpp +++ b/src/mlpack/tests/ann/layer/parametric_relu.cpp @@ -83,4 +83,4 @@ TEST_CASE("PReLUGRADIENTTest", "[ANNLayerTest]") REQUIRE(0.0103 - arma::accu(predGradient) == Approx(0.0).margin(1e-4)); -} \ No newline at end of file +} From 84e86cb709a66b82ad2d38847b71a208bae6c0e9 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Fri, 17 Feb 2023 10:55:56 +0530 Subject: [PATCH 73/88] fix conflict --- .../methods/cf/decomposition_policies/decomposition_policies.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp b/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp index ca3df3af0f..4f0dab343e 100644 --- a/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp +++ b/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp @@ -20,6 +20,7 @@ #include "svd_complete_method.hpp" #include "svd_incomplete_method.hpp" #include "svdplusplus_method.hpp" +#include "quic_svd_method.hpp" #include "block_krylov_svd_method.hpp" #endif From 5b4bedd80a638466f3da96efab289d55ca6b1d60 Mon Sep 17 00:00:00 2001 From: AdarshSantoria Date: Fri, 17 Feb 2023 12:24:01 +0530 Subject: [PATCH 74/88] add changes to binding --- src/mlpack/methods/cf/cf_main.cpp | 17 ++++++- src/mlpack/methods/cf/cf_model.hpp | 4 +- src/mlpack/methods/cf/cf_model_impl.hpp | 47 ++++++++++--------- .../decomposition_policies.hpp | 2 +- 4 files changed, 44 insertions(+), 26 deletions(-) diff --git a/src/mlpack/methods/cf/cf_main.cpp b/src/mlpack/methods/cf/cf_main.cpp index 1378d49ef6..b0f816a5c7 100644 --- a/src/mlpack/methods/cf/cf_main.cpp +++ b/src/mlpack/methods/cf/cf_main.cpp @@ -72,6 +72,9 @@ BINDING_LONG_DESC( " - 'SVDCompleteIncremental' -- SVD complete incremental learning\n" " - 'BiasSVD' -- Bias SVD using a SGD optimizer\n" " - 'SVDPP' -- SVD++ using a SGD optimizer\n" + " - 'RandSVD' -- RandomizedSVD learning\n" + " - 'QSVD' -- QuicSVD learning\n" + " - 'BKSVD' -- Block Krylov SVD\n" "\n\n" "The following neighbor search algorithms can be specified via" + " the " + PRINT_PARAM_STRING("neighbor_search") + " parameter:" @@ -196,7 +199,7 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers) RequireParamInSet(params, "algorithm", { "NMF", "BatchSVD", "SVDIncompleteIncremental", "SVDCompleteIncremental", "RegSVD", - "RandSVD", "BiasSVD", "SVDPP" }, true, "unknown algorithm"); + "RandSVD", "BiasSVD", "SVDPP", "QSVD", "BKSVD" }, true, "unknown algorithm"); ReportIgnoredParam(params, {{ "iteration_only_termination", true }}, "min_residue"); @@ -282,6 +285,18 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers) "when max_iterations is reached"); cf->DecompositionType() = CFModel::SVD_PLUS_PLUS; } + else if (algo == "QSVD") + { + ReportIgnoredParam(params, "min_residue", "QSVD terminates only " + "when max_iterations is reached"); + cf->DecompositionType() = CFModel::QUIC_SVD; + } + else if (algo == "BKSVD") + { + ReportIgnoredParam(params, "min_residue", "BKSVD terminates only " + "when max_iterations is reached"); + cf->DecompositionType() = CFModel::BLOCK_KRYLOV_SVD; + } // Perform the factorization and do whatever the user wanted. const size_t neighborhood = (size_t) params.Get("neighborhood"); diff --git a/src/mlpack/methods/cf/cf_model.hpp b/src/mlpack/methods/cf/cf_model.hpp index 30be735ddd..dde86162ae 100644 --- a/src/mlpack/methods/cf/cf_model.hpp +++ b/src/mlpack/methods/cf/cf_model.hpp @@ -170,7 +170,9 @@ class CFModel SVD_COMPLETE, SVD_INCOMPLETE, BIAS_SVD, - SVD_PLUS_PLUS + SVD_PLUS_PLUS, + QUIC_SVD, + BLOCK_KRYLOV_SVD }; enum NormalizationTypes diff --git a/src/mlpack/methods/cf/cf_model_impl.hpp b/src/mlpack/methods/cf/cf_model_impl.hpp index d7f40b1b90..8b1d05e0ca 100644 --- a/src/mlpack/methods/cf/cf_model_impl.hpp +++ b/src/mlpack/methods/cf/cf_model_impl.hpp @@ -14,29 +14,6 @@ #include "cf_model.hpp" -#include "interpolation_policies/average_interpolation.hpp" -#include "interpolation_policies/regression_interpolation.hpp" -#include "interpolation_policies/similarity_interpolation.hpp" - -#include "neighbor_search_policies/cosine_search.hpp" -#include "neighbor_search_policies/lmetric_search.hpp" -#include "neighbor_search_policies/pearson_search.hpp" - -#include "decomposition_policies/batch_svd_method.hpp" -#include "decomposition_policies/bias_svd_method.hpp" -#include "decomposition_policies/nmf_method.hpp" -#include "decomposition_policies/randomized_svd_method.hpp" -#include "decomposition_policies/regularized_svd_method.hpp" -#include "decomposition_policies/svd_complete_method.hpp" -#include "decomposition_policies/svd_incomplete_method.hpp" -#include "decomposition_policies/svdplusplus_method.hpp" - -#include "normalization/no_normalization.hpp" -#include "normalization/overall_mean_normalization.hpp" -#include "normalization/user_mean_normalization.hpp" -#include "normalization/item_mean_normalization.hpp" -#include "normalization/z_score_normalization.hpp" - namespace mlpack { inline CFModel::CFModel() : @@ -361,6 +338,12 @@ inline CFWrapperBase* InitializeModel( case CFModel::SVD_PLUS_PLUS: return InitializeModelHelper(normalizationType); + + case CFModel::QUIC_SVD: + return InitializeModelHelper(normalizationType); + + case CFModel::BLOCK_KRYLOV_SVD: + return InitializeModelHelper(normalizationType); } // This shouldn't ever happen. @@ -473,6 +456,16 @@ inline void CFModel::Train( cf = TrainHelper(SVDPlusPlusPolicy(), normalizationType, data, numUsersForSimilarity, rank, maxIterations, minResidue, mit); break; + + case QUIC_SVD: + cf = TrainHelper(QUIC_SVDPolicy(), normalizationType, data, + numUsersForSimilarity, rank, maxIterations, minResidue, mit); + break; + + case BLOCK_KRYLOV_SVD: + cf = TrainHelper(BlockKrylovSVDPolicy(), normalizationType, data, + numUsersForSimilarity, rank, maxIterations, minResidue, mit); + break; } } @@ -555,6 +548,14 @@ void CFModel::serialize(Archive& ar, const uint32_t /* version */) case SVD_PLUS_PLUS: SerializeHelper(ar, cf, normalizationType); break; + + case QUIC_SVD: + SerializeHelper(ar, cf, normalizationType); + break; + + case BLOCK_KRYLOV_SVD: + SerializeHelper(ar, cf, normalizationType); + break; } } diff --git a/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp b/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp index 4f0dab343e..bb5788dc1b 100644 --- a/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp +++ b/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp @@ -1,5 +1,5 @@ /** - * @file methods/cf/decomposition_policies/decomposition_policies.hpp + * @file #include "decomposition_policies/decomposition_policies.hpp" * @author Ryan Curtin * * Convenience include for all decomposition policies implemented for CF. From 613ab5fbfbda9c7ad2df69436b079998cb1ffdd7 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Fri, 17 Feb 2023 13:11:41 +0530 Subject: [PATCH 75/88] typo fix --- src/mlpack/methods/cf/cf_main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/cf/cf_main.cpp b/src/mlpack/methods/cf/cf_main.cpp index b0f816a5c7..80107ee03d 100644 --- a/src/mlpack/methods/cf/cf_main.cpp +++ b/src/mlpack/methods/cf/cf_main.cpp @@ -74,7 +74,7 @@ BINDING_LONG_DESC( " - 'SVDPP' -- SVD++ using a SGD optimizer\n" " - 'RandSVD' -- RandomizedSVD learning\n" " - 'QSVD' -- QuicSVD learning\n" - " - 'BKSVD' -- Block Krylov SVD\n" + " - 'BKSVD' -- Block Krylov SVD learning\n" "\n\n" "The following neighbor search algorithms can be specified via" + " the " + PRINT_PARAM_STRING("neighbor_search") + " parameter:" From 72bc449cfe36dab6e76d73dad6772a5c10831d58 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Fri, 17 Feb 2023 13:14:27 +0530 Subject: [PATCH 76/88] typo fix --- .../cf/decomposition_policies/decomposition_policies.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp b/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp index bb5788dc1b..4f0dab343e 100644 --- a/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp +++ b/src/mlpack/methods/cf/decomposition_policies/decomposition_policies.hpp @@ -1,5 +1,5 @@ /** - * @file #include "decomposition_policies/decomposition_policies.hpp" + * @file methods/cf/decomposition_policies/decomposition_policies.hpp * @author Ryan Curtin * * Convenience include for all decomposition policies implemented for CF. From 531f06501457de3674215147d662b8db4ad87c8d Mon Sep 17 00:00:00 2001 From: AdarshSantoria Date: Sun, 19 Feb 2023 20:06:58 +0530 Subject: [PATCH 77/88] code fix --- .../ann/layer/parametric_relu_impl.hpp | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp index 5a8af8da14..e557b54638 100644 --- a/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp +++ b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp @@ -22,31 +22,28 @@ namespace mlpack { template PReLUType::PReLUType(const double userAlpha) : - Layer(), - userAlpha(userAlpha) + Layer() { - alpha.set_size(WeightSize(), 1); + alpha.set_size(1, 1); alpha(0) = userAlpha; } template PReLUType::PReLUType( const PReLUType& other) : - Layer(other), - userAlpha(other.userAlpha) + Layer(other) { - alpha.set_size(WeightSize(), 1); - alpha(0) = userAlpha; + userAlpha = other.userAlpha; + alpha = other.alpha; } template PReLUType::PReLUType( PReLUType&& other) : - Layer(std::move(other)), - userAlpha(std::move(other.userAlpha)) + Layer(std::move(other)) { - alpha.set_size(WeightSize(), 1); - alpha(0) = userAlpha; + userAlpha = std::move(other.userAlpha); + alpha = std::move(other.alpha); } template @@ -86,7 +83,6 @@ void PReLUType::SetWeights( //! Set value of alpha to the one given by user. // TODO: this doesn't even make any sense. is it trainable or not? // why is there userAlpha? is that for initialization only? - alpha(0) = userAlpha; } template @@ -95,9 +91,12 @@ void PReLUType::Forward( { // TODO: use transform()? output = input; - #pragma omp for - for (size_t i = 0; i < input.n_elem; ++i) - output(i) *= (input(i) >= 0) ? 1 : alpha(0); + if (this->training) + { + #pragma omp for + for (size_t i = 0; i < input.n_elem; ++i) + output(i) *= (input(i) >= 0) ? 1 : alpha(0); + } } template @@ -120,7 +119,7 @@ void PReLUType::Gradient( MatType& gradient) { MatType zeros = arma::zeros(input.n_rows, input.n_cols); - gradient.set_size(WeightSize(), 1); + gradient.set_size(1, 1); gradient(0) = arma::accu(error % arma::min(zeros, input)) / input.n_cols; } @@ -132,6 +131,7 @@ void PReLUType::serialize( { ar(cereal::base_class>(this)); + ar(CEREAL_NVP(userAlpha)); ar(CEREAL_NVP(alpha)); } From c44e6944940bbc6c75da1fd5c604a40f3676afd2 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Mon, 20 Feb 2023 21:13:41 +0530 Subject: [PATCH 78/88] Update HISTORY.md --- HISTORY.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 3976c0d6d3..009e4868a7 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,8 @@ ### mlpack ?.?.? ###### ????-??-?? + * Fix PReLU ann layer (#3420). + * Update outdated code in tutorials (#3398). * Bugfix for non-square convolution kernels (#3376). From 7d67b49d5c88f93fac823df832b01b8150a752fb Mon Sep 17 00:00:00 2001 From: AdarshSantoria Date: Wed, 22 Feb 2023 17:20:35 +0530 Subject: [PATCH 79/88] add initialization and serialization --- src/mlpack/methods/ann/layer/parametric_relu.hpp | 10 ++++++++++ .../methods/ann/layer/parametric_relu_impl.hpp | 12 ++++++++++++ src/mlpack/methods/ann/layer/serialization.hpp | 1 + 3 files changed, 23 insertions(+) diff --git a/src/mlpack/methods/ann/layer/parametric_relu.hpp b/src/mlpack/methods/ann/layer/parametric_relu.hpp index 6edd851059..f8f99e3ee5 100644 --- a/src/mlpack/methods/ann/layer/parametric_relu.hpp +++ b/src/mlpack/methods/ann/layer/parametric_relu.hpp @@ -70,6 +70,16 @@ class PReLUType : public Layer //! Reset the layer parameter. void SetWeights(typename MatType::elem_type* weightsPtr); + /** + * Initialize the weight matrix of the layer. + * + * @param W Weight matrix to initialize. + * @param elements Number of elements. + */ + void CustomInitialize( + MatType& W, + const size_t elements); + /** * Ordinary feed forward pass of a neural network, evaluating the function * f(x) by propagating the activity forward through f. diff --git a/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp index e557b54638..679b39505d 100644 --- a/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp +++ b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp @@ -85,6 +85,18 @@ void PReLUType::SetWeights( // why is there userAlpha? is that for initialization only? } +template +void PReLUType::CustomInitialize( + MatType& W, + const size_t elements) +{ + if (elements != 1) { + throw std::invalid_argument("PReLUType::CustomInitialize(): wrong " + "elements size!"); + } + MakeAlias(alpha, W.memptr(), 1, 1); +} + template void PReLUType::Forward( const MatType& input, MatType& output) diff --git a/src/mlpack/methods/ann/layer/serialization.hpp b/src/mlpack/methods/ann/layer/serialization.hpp index 90b785394c..99cb7cc823 100644 --- a/src/mlpack/methods/ann/layer/serialization.hpp +++ b/src/mlpack/methods/ann/layer/serialization.hpp @@ -65,6 +65,7 @@ CEREAL_REGISTER_TYPE(mlpack::MeanPoolingType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::NoisyLinearType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::PaddingType<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::PReLUType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::RBFType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::SoftmaxType<__VA_ARGS__>); \ From 94dd26f9297b9cf9bc0b045d919046660966adfc Mon Sep 17 00:00:00 2001 From: AdarshSantoria Date: Mon, 27 Feb 2023 00:48:37 +0530 Subject: [PATCH 80/88] add notadapted tests for ann --- src/mlpack/tests/ann/layer/ann_layer_test.cpp | 4538 +++++++++++++++++ src/mlpack/tests/ann/layer/concat.cpp | 227 + src/mlpack/tests/ann/layer/concatenate.cpp | 95 + src/mlpack/tests/ann/layer_test.cpp | 2 + src/mlpack/tests/ann/loss_functions_test.cpp | 21 + 5 files changed, 4883 insertions(+) create mode 100644 src/mlpack/tests/ann/layer/ann_layer_test.cpp create mode 100644 src/mlpack/tests/ann/layer/concat.cpp create mode 100644 src/mlpack/tests/ann/layer/concatenate.cpp diff --git a/src/mlpack/tests/ann/layer/ann_layer_test.cpp b/src/mlpack/tests/ann/layer/ann_layer_test.cpp new file mode 100644 index 0000000000..cfd9ec0464 --- /dev/null +++ b/src/mlpack/tests/ann/layer/ann_layer_test.cpp @@ -0,0 +1,4538 @@ +/** + * @file tests/ann_layer_test.cpp + * @author Marcus Edel + * @author Praveen Ch + * + * Tests the ann layer modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include +#include + +#include "../test_catch_tools.hpp" +#include "../catch.hpp" +#include "../serialization.hpp" +#include "ann_test_tools.hpp" + +using namespace mlpack; + +// // network1 should be allocated with `new`, and trained on some data. +// template +// void CheckRNNCopyFunction(ModelType* network1, +// MatType& trainData, +// MatType& trainLabels, +// const size_t maxEpochs) +// { +// arma::cube predictions1; +// arma::cube predictions2; +// ens::StandardSGD opt(0.1, 1, maxEpochs * trainData.n_slices, -100, false); + +// network1->Train(trainData, trainLabels, opt); +// network1->Predict(trainData, predictions1); + +// RNN<> network2 = *network1; +// delete network1; + +// // Deallocating all of network1's memory, so that network2 does not use any +// // of that memory. +// network2.Predict(trainData, predictions2); +// CheckMatrices(predictions1, predictions2); +// } + +// // network1 should be allocated with `new`, and trained on some data. +// template +// void CheckRNNMoveFunction(ModelType* network1, +// MatType& trainData, +// MatType& trainLabels, +// const size_t maxEpochs) +// { +// arma::cube predictions1; +// arma::cube predictions2; +// ens::StandardSGD opt(0.1, 1, maxEpochs * trainData.n_slices, -100, false); + +// network1->Train(trainData, trainLabels, opt); +// network1->Predict(trainData, predictions1); + +// RNN<> network2(std::move(*network1)); +// delete network1; + +// // Deallocating all of network1's memory, so that network2 does not use any +// // of that memory. +// network2.Predict(trainData, predictions2); +// CheckMatrices(predictions1, predictions2); +// } + +/** + * Simple add module test. + * +TEST_CASE("SimpleAddLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + Add module(10); + module.Parameters().randu(); + + // Test the Forward function. + input = arma::zeros(10, 1); + module.Forward(input, output); + REQUIRE(arma::accu(module.Parameters()) == arma::accu(output)); + + // Test the Backward function. + module.Backward(input, output, delta); + REQUIRE(arma::accu(output) == arma::accu(delta)); + + // Test the forward function. + input = arma::ones(10, 1); + module.Forward(input, output); + REQUIRE(10 + arma::accu(module.Parameters()) == + Approx(arma::accu(output)).epsilon(1e-5)); + + // Test the backward function. + module.Backward(input, output, delta); + REQUIRE(arma::accu(output) == Approx(arma::accu(delta)).epsilon(1e-5)); +} +*/ + +/** + * Jacobian add module test. + * +TEST_CASE("JacobianAddLayerTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t elements = RandInt(2, 1000); + arma::mat input; + input.set_size(elements, 1); + + Add module(elements); + module.Parameters().randu(); + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} +*/ + +/** + * Add layer numerical gradient test. + * +TEST_CASE("GradientAddLayerTest", "[ANNLayerTest]") +{ + // Add function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(10, 1)), + target(arma::mat("0")) + { + model = new FFN(); + model->ResetData(input, target); + model->Add(); + model->Add(10, 10); + model->Add(10); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + REQUIRE(CheckGradient(function) <= 1e-4); +}*/ + +/** + * Test that the function that can access the outSize parameter of + * the Add layer works. + * +TEST_CASE("AddLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter : outSize. + Add layer(7); + + // Make sure we can get the parameter successfully. + REQUIRE(layer.OutputSize() == 7); +}*/ + +/** + * Simple constant module test. + * +TEST_CASE("SimpleConstantLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + Constant module(10, 3.0); + + // Test the Forward function. + input = arma::zeros(10, 1); + module.Forward(input, output); + REQUIRE(arma::accu(output) == 30.0); + + // Test the Backward function. + module.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 0); + + // Test the forward function. + input = arma::ones(10, 1); + module.Forward(input, output); + REQUIRE(arma::accu(output) == 30.0); + + // Test the backward function. + module.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 0); +}*/ + +/** + * Jacobian constant module test. + * +TEST_CASE("JacobianConstantLayerTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t elements = RandInt(2, 1000); + arma::mat input; + input.set_size(elements, 1); + + Constant module(elements, 1.0); + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +}*/ + +/** + * Test that the function that can access the outSize parameter of the + * Constant layer works. + * +TEST_CASE("ConstantLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter : outSize. + Constant layer(7); + + // Make sure we can get the parameter successfully. + REQUIRE(layer.OutSize() == 7); +}*/ + +// /** +// * Simple linear module test. +// */ +// TEST_CASE("SimpleLinearLayerTest", "[ANNLayerTest]") +// { +// arma::mat output, input, delta; +// Linear<> module(10, 10); +// module.Parameters().randu(); +// module.Reset(); + +// // Test the Forward function. +// input = arma::zeros(10, 1); +// module.Forward(input, output); +// REQUIRE(arma::accu(module.Parameters().submat(100, +// 0, module.Parameters().n_elem - 1, 0)) == +// Approx(arma::accu(output)).epsilon(1e-5)); + +// // Test the Backward function. +// module.Backward(input, input, delta); +// REQUIRE(arma::accu(delta) == 0); +// } + +// /** +// * Jacobian linear module test. +// */ +// TEST_CASE("JacobianLinearLayerTest", "[ANNLayerTest]") +// { +// for (size_t i = 0; i < 5; ++i) +// { +// const size_t inputElements = RandInt(2, 1000); +// const size_t outputElements = RandInt(2, 1000); + +// arma::mat input; +// input.set_size(inputElements, 1); + +// Linear<> module(inputElements, outputElements); +// module.Parameters().randu(); + +// double error = JacobianTest(module, input); +// REQUIRE(error <= 1e-5); +// } +// } + +// /** +// * Linear layer numerical gradient test. +// */ +// TEST_CASE("GradientLinearLayerTest", "[ANNLayerTest]") +// { +// // Linear function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(10, 1)), +// target(arma::mat("1")) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(10, 10); +// model->Add >(10, 2); +// model->Add >(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// FFN* model; +// arma::mat input, target; +// } function; + +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +// /** +// * Simple noisy linear module test. +// */ +// TEST_CASE("SimpleNoisyLinearLayerTest", "[ANNLayerTest]") +// { +// arma::mat output, input, delta; +// NoisyLinear<> module(10, 10); +// module.Parameters().randu(); +// module.Reset(); + +// // Test the Backward function. +// module.Backward(input, input, delta); +// REQUIRE(arma::accu(delta) == 0); +// } + +// /** +// * Jacobian noisy linear module test. +// */ +// TEST_CASE("JacobianNoisyLinearLayerTest", "[ANNLayerTest]") +// { +// const size_t inputElements = RandInt(2, 1000); +// const size_t outputElements = RandInt(2, 1000); + +// arma::mat input; +// input.set_size(inputElements, 1); + +// NoisyLinear<> module(inputElements, outputElements); +// module.Parameters().randu(); + +// double error = JacobianTest(module, input); +// REQUIRE(error <= 1e-5); +// } + +// /** +// * Noisy Linear layer numerical gradient test. +// */ +// TEST_CASE("GradientNoisyLinearLayerTest", "[ANNLayerTest]") +// { +// // Noisy linear function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(10, 1)), +// target(arma::mat("1")) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(10, 10); +// model->Add >(10, 2); +// model->Add >(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// FFN* model; +// arma::mat input, target; +// } function; + +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +/** + * Jacobian LeakyReLU module test. + * +TEST_CASE("JacobianLeakyReLULayerTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t inputElements = RandInt(2, 1000); + + arma::mat input; + input.set_size(inputElements, 1); + + LeakyReLU module; + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} +*/ + +/** + * Jacobian FlexibleReLU module test. + * +TEST_CASE("JacobianFlexibleReLULayerTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t inputElements = RandInt(2, 1000); + + arma::mat input; + input.set_size(inputElements, 1); + + FlexibleReLU module; + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} +*/ + +/** + * Flexible ReLU layer numerical gradient test. + * +TEST_CASE("GradientFlexibleReLULayerTest", "[ANNLayerTest]") +{ + // Add function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(2, 1)), + target(arma::mat("0")) + { + model = new FFN( + NegativeLogLikelihood(), RandomInitialization(0.1, 0.5)); + + model->ResetData(input, target); + model->Add(2, 2); + model->Add(2, 5); + model->Add(0.05); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + REQUIRE(CheckGradient(function) <= 1e-4); +} +*/ + +/** + * Jacobian MultiplyConstant module test. + * +TEST_CASE("JacobianMultiplyConstantLayerTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t inputElements = RandInt(2, 1000); + + arma::mat input; + input.set_size(inputElements, 1); + + MultiplyConstant module(3.0); + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} +*/ + +/** + * Check whether copying and moving network with MultiplyConstant is working or + * not. + */ +// TEST_CASE("CheckCopyMoveMultiplyConstantTest", "[ANNLayerTest]") +// { +// arma::mat input(2, 1000); +// input.randu(); +// +// arma::mat output1; +// arma::mat output2; +// arma::mat output3; +// arma::mat output4; +// +// MultiplyConstant<> *module1 = new MultiplyConstant<>(3.0); +// module1->Forward(input, output1); +// +// MultiplyConstant<> module2 = *module1; +// delete module1; +// +// module2.Forward(input, output2); +// CheckMatrices(output1, output2); +// +// MultiplyConstant<> *module3 = new MultiplyConstant<>(3.0); +// module3->Forward(input, output3); +// +// MultiplyConstant<> module4(std::move(*module3)); +// delete module3; +// +// module4.Forward(input, output4); +// CheckMatrices(output3, output4); +// } + +/** + * Jacobian HardTanH module test. + * +TEST_CASE("JacobianHardTanHLayerTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t inputElements = RandInt(2, 1000); + + arma::mat input; + input.set_size(inputElements, 1); + + HardTanH module; + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} +*/ + +/** + * Simple select module test. + * +TEST_CASE("SimpleSelectLayerTest", "[ANNLayerTest]") +{ + // TODO: this needs to be adapted + arma::mat outputA, outputB, input, delta; + + input = arma::ones(10, 5); + for (size_t i = 0; i < input.n_cols; ++i) + { + input.col(i) *= i; + } + + // Test the Forward function. + Select moduleA(3); + moduleA.Forward(input, outputA); + REQUIRE(30 == arma::accu(outputA)); + + // Test the Forward function. + Select moduleB(3, 5); + moduleB.Forward(input, outputB); + REQUIRE(15 == arma::accu(outputB)); + + // Test the Backward function. + moduleA.Backward(input, outputA, delta); + REQUIRE(30 == arma::accu(delta)); + + // Test the Backward function. + moduleB.Backward(input, outputA, delta); + REQUIRE(15 == arma::accu(delta)); +} +*/ + +/** + * Test that the functions that can access the parameters of the + * Select layer work. + * +TEST_CASE("SelectLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : index, elements. + Select layer(3, 5); + + // Make sure we can get the parameters successfully. + REQUIRE(layer.Index() == 3); + REQUIRE(layer.NumElements() == 5); +} +*/ + +/** + * Simple join module test. + * +TEST_CASE("SimpleJoinLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + input = arma::ones(10, 5); + + // Test the Forward function. + Join module; + module.Forward(input, output); + REQUIRE(50 == arma::accu(output)); + + bool b = output.n_rows == 1 || output.n_cols == 1; + REQUIRE(b == true); + + // Test the Backward function. + module.Backward(input, output, delta); + REQUIRE(50 == arma::accu(delta)); + + b = delta.n_rows == input.n_rows && input.n_cols; + REQUIRE(b == true); +} +*/ + +// /** +// * Simple add merge module test. +// */ +// TEST_CASE("SimpleAddMergeLayerTest", "[ANNLayerTest]") +// { +// arma::mat output, input, delta; +// input = arma::ones(10, 1); + +// for (size_t i = 0; i < 5; ++i) +// { +// AddMerge<> module(false, false); +// const size_t numMergeModules = RandInt(2, 10); +// for (size_t m = 0; m < numMergeModules; ++m) +// { +// IdentityLayer<> identityLayer; +// identityLayer.Forward(input, identityLayer.OutputParameter()); + +// module.Add >(identityLayer); +// } + +// // Test the Forward function. +// module.Forward(input, output); +// REQUIRE(10 * numMergeModules == arma::accu(output)); + +// // Test the Backward function. +// module.Backward(input, output, delta); +// REQUIRE(arma::accu(output) == arma::accu(delta)); +// } +// } + +// /** +// * Test the LSTM layer with a user defined rho parameter and without. +// */ +// TEST_CASE("LSTMRrhoTest", "[ANNLayerTest]") +// { +// const size_t rho = 5; +// arma::cube input = arma::randu(1, 1, 5); +// arma::cube target = arma::ones(1, 1, 5); +// RandomInitialization init(0.5, 0.5); + +// // Create model with user defined rho parameter. +// RNN modelA( +// rho, false, NegativeLogLikelihood(), init); +// modelA.Add >(); +// modelA.Add >(1, 10); + +// // Use LSTM layer with rho. +// modelA.Add >(10, 3, rho); +// modelA.Add >(); + +// // Create model without user defined rho parameter. +// RNN modelB( +// rho, false, NegativeLogLikelihood(), init); +// modelB.Add >(); +// modelB.Add >(1, 10); + +// // Use LSTM layer with rho = MAXSIZE. +// modelB.Add >(10, 3); +// modelB.Add >(); + +// ens::StandardSGD opt(0.1, 1, 5, -100, false); +// modelA.Train(input, target, opt); +// modelB.Train(input, target, opt); + +// CheckMatrices(modelB.Parameters(), modelA.Parameters()); +// } + +// /** +// * LSTM layer numerical gradient test. +// */ +// TEST_CASE("GradientLSTMLayerTest", "[ANNLayerTest]") +// { +// // LSTM function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(1, 1, 5)), +// target(arma::ones(1, 1, 5)) +// { +// const size_t rho = 5; + +// model = new RNN(rho); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(1, 10); +// model->Add >(10, 3, rho); +// model->Add >(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// RNN* model; +// arma::cube input, target; +// } function; + +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +// /** +// * Test that the functions that can modify and access the parameters of the +// * LSTM layer work. +// */ +// TEST_CASE("LSTMLayerParametersTest", "[ANNLayerTest]") +// { +// // Parameter order : inSize, outSize, rho. +// LSTM<> layer1(1, 2, 3); +// LSTM<> layer2(1, 2, 4); + +// // Make sure we can get the parameters successfully. +// REQUIRE(layer1.InSize() == 1); +// REQUIRE(layer1.OutSize() == 2); +// REQUIRE(layer1.Rho() == 3); + +// // Now modify the parameters to match the second layer. +// layer1.Rho() = 4; + +// // Now ensure all the results are the same. +// REQUIRE(layer1.InSize() == layer2.InSize()); +// REQUIRE(layer1.OutSize() == layer2.OutSize()); +// REQUIRE(layer1.Rho() == layer2.Rho()); +// } + +// /** +// * Test the FastLSTM layer with a user defined rho parameter and without. +// */ +// TEST_CASE("FastLSTMRrhoTest", "[ANNLayerTest]") +// { +// const size_t rho = 5; +// arma::cube input = arma::randu(1, 1, 5); +// arma::cube target = arma::ones(1, 1, 5); +// RandomInitialization init(0.5, 0.5); + +// // Create model with user defined rho parameter. +// RNN modelA( +// rho, false, NegativeLogLikelihood(), init); +// modelA.Add >(); +// modelA.Add >(1, 10); + +// // Use FastLSTM layer with rho. +// modelA.Add >(10, 3, rho); +// modelA.Add >(); + +// // Create model without user defined rho parameter. +// RNN modelB( +// rho, false, NegativeLogLikelihood(), init); +// modelB.Add >(); +// modelB.Add >(1, 10); + +// // Use FastLSTM layer with rho = MAXSIZE. +// modelB.Add >(10, 3); +// modelB.Add >(); + +// ens::StandardSGD opt(0.1, 1, 5, -100, false); +// modelA.Train(input, target, opt); +// modelB.Train(input, target, opt); + +// CheckMatrices(modelB.Parameters(), modelA.Parameters()); +// } + +// /** +// * FastLSTM layer numerical gradient test. +// */ +// TEST_CASE("GradientFastLSTMLayerTest", "[ANNLayerTest]") +// { +// // Fast LSTM function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(1, 1, 5)), +// target(arma::ones(1, 1, 5)) +// { +// const size_t rho = 5; + +// model = new RNN(rho); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(1, 10); +// model->Add >(10, 3, rho); +// model->Add >(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// RNN* model; +// arma::cube input, target; +// } function; + +// // The threshold should be << 0.1 but since the Fast LSTM layer uses an +// // approximation of the sigmoid function the estimated gradient is not +// // correct. +// REQUIRE(CheckGradient(function) <= 0.2); +// } + +// /** +// * Test that the functions that can modify and access the parameters of the +// * Fast LSTM layer work. +// */ +// TEST_CASE("FastLSTMLayerParametersTest", "[ANNLayerTest]") +// { +// // Parameter order : inSize, outSize, rho. +// FastLSTM<> layer1(1, 2, 3); +// FastLSTM<> layer2(1, 2, 4); + +// // Make sure we can get the parameters successfully. +// REQUIRE(layer1.InSize() == 1); +// REQUIRE(layer1.OutSize() == 2); +// REQUIRE(layer1.Rho() == 3); + +// // Now modify the parameters to match the second layer. +// layer1.Rho() = 4; + +// // Now ensure all the results are the same. +// REQUIRE(layer1.InSize() == layer2.InSize()); +// REQUIRE(layer1.OutSize() == layer2.OutSize()); +// REQUIRE(layer1.Rho() == layer2.Rho()); +// } + +// /** +// * Check whether copying and moving network with FastLSTM is working or not. +// */ +// TEST_CASE("CheckCopyMoveFastLSTMTest", "[ANNLayerTest]") +// { +// arma::cube input = arma::randu(1, 1, 5); +// arma::cube target = arma::ones(1, 1, 5); +// const size_t rho = 5; + +// RNN *model1 = +// new RNN(rho); +// model1->ResetData(input, target); +// model1->Add >(); +// model1->Add >(1, 10); +// model1->Add >(10, 3, rho); +// model1->Add >(); + +// RNN *model2 = +// new RNN(rho); +// model2->ResetData(input, target); +// model2->Add >(); +// model2->Add >(1, 10); +// model2->Add >(10, 3, rho); +// model2->Add >(); + +// // Check whether copy constructor is working or not. +// CheckRNNCopyFunction<>(model1, input, target, 1); + +// // Check whether move constructor is working or not. +// CheckRNNMoveFunction<>(model2, input, target, 1); +// } + +// /** +// * Check whether copying and moving network with LSTM is working or not. +// */ +// TEST_CASE("CheckCopyMoveLSTMTest", "[ANNLayerTest]") +// { +// arma::cube input = arma::randu(1, 1, 5); +// arma::cube target = arma::ones(1, 1, 5); +// const size_t rho = 5; + +// RNN *model1 = +// new RNN(rho); +// model1->ResetData(input, target); +// model1->Add >(); +// model1->Add >(1, 10); +// model1->Add >(10, 3, rho); +// model1->Add >(); + +// RNN *model2 = +// new RNN(rho); +// model2->ResetData(input, target); +// model2->Add >(); +// model2->Add >(1, 10); +// model2->Add >(10, 3, rho); +// model2->Add >(); + +// // Check whether copy constructor is working or not. +// CheckRNNCopyFunction<>(model1, input, target, 1); + +// // Check whether move constructor is working or not. +// CheckRNNMoveFunction<>(model2, input, target, 1); +// } + +// /** +// * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell +// * state. Besides output, the overloaded function provides read access to cell +// * state of the LSTM layer. +// */ +// TEST_CASE("ReadCellStateParamLSTMLayerTest", "[ANNLayerTest]") +// { +// const size_t rho = 5, inputSize = 3, outputSize = 2; + +// // Provide input of all ones. +// arma::cube input = arma::ones(inputSize, outputSize, rho); + +// arma::mat inputGate, forgetGate, outputGate, hidden; +// arma::mat outLstm, cellLstm; + +// // LSTM layer. +// LSTM<> lstm(inputSize, outputSize, rho); +// lstm.Reset(); +// lstm.ResetCell(rho); + +// // Initialize the weights to all ones. +// lstm.Parameters().ones(); + +// arma::mat inputWeight = arma::ones(outputSize, inputSize); +// arma::mat outputWeight = arma::ones(outputSize, outputSize); +// arma::mat bias = arma::ones(outputSize, input.n_cols); +// arma::mat cellCalc = arma::zeros(outputSize, input.n_cols); +// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); + +// for (size_t seqNum = 0; seqNum < rho; ++seqNum) +// { +// // Wrap a matrix around our data to avoid a copy. +// arma::mat stepData(input.slice(seqNum).memptr(), +// input.n_rows, input.n_cols, false, true); + +// // Apply Forward() on LSTM layer. +// lstm.Forward(stepData, // Input. +// outLstm, // Output. +// cellLstm, // Cell state. +// false); // Don't write into the cell state. + +// // Compute the value of cell state and output. +// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); + +// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); + +// // z = tanh(W.dot(x) + W.dot(h) + b). +// hidden = arma::tanh(inputWeight * stepData + +// outputWeight * outCalc + bias); + +// // c = f * c + i * z. +// cellCalc = forgetGate % cellCalc + inputGate % hidden; + +// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); + +// // h = o * tanh(c). +// outCalc = outputGate % arma::tanh(cellCalc); + +// CheckMatrices(outLstm, outCalc, 1e-12); +// CheckMatrices(cellLstm, cellCalc, 1e-12); +// } +// } + +// /** +// * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell +// * state. Besides output, the overloaded function provides write access to cell +// * state of the LSTM layer. +// */ +// TEST_CASE("WriteCellStateParamLSTMLayerTest", "[ANNLayerTest]") +// { +// const size_t rho = 5, inputSize = 3, outputSize = 2; + +// // Provide input of all ones. +// arma::cube input = arma::ones(inputSize, outputSize, rho); + +// arma::mat inputGate, forgetGate, outputGate, hidden; +// arma::mat outLstm, cellLstm; +// arma::mat cellCalc; + +// // LSTM layer. +// LSTM<> lstm(inputSize, outputSize, rho); +// lstm.Reset(); +// lstm.ResetCell(rho); + +// // Initialize the weights to all ones. +// lstm.Parameters().ones(); + +// arma::mat inputWeight = arma::ones(outputSize, inputSize); +// arma::mat outputWeight = arma::ones(outputSize, outputSize); +// arma::mat bias = arma::ones(outputSize, input.n_cols); +// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); + +// for (size_t seqNum = 0; seqNum < rho; ++seqNum) +// { +// // Wrap a matrix around our data to avoid a copy. +// arma::mat stepData(input.slice(seqNum).memptr(), +// input.n_rows, input.n_cols, false, true); + +// if (cellLstm.is_empty()) +// { +// // Set the cell state to zeros. +// cellLstm = arma::zeros(outputSize, input.n_cols); +// cellCalc = arma::zeros(outputSize, input.n_cols); +// } +// else +// { +// // Set the cell state to zeros. +// cellLstm = arma::zeros(cellLstm.n_rows, cellLstm.n_cols); +// cellCalc = arma::zeros(cellCalc.n_rows, cellCalc.n_cols); +// } + +// // Apply Forward() on the LSTM layer. +// lstm.Forward(stepData, // Input. +// outLstm, // Output. +// cellLstm, // Cell state. +// true); // Write into cell state. + +// // Compute the value of cell state and output. +// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); + +// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); + +// // z = tanh(W.dot(x) + W.dot(h) + b). +// hidden = arma::tanh(inputWeight * stepData + +// outputWeight * outCalc + bias); + +// // c = f * c + i * z. +// cellCalc = forgetGate % cellCalc + inputGate % hidden; + +// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); + +// // h = o * tanh(c). +// outCalc = outputGate % arma::tanh(cellCalc); + +// CheckMatrices(outLstm, outCalc, 1e-12); +// CheckMatrices(cellLstm, cellCalc, 1e-12); +// } + +// // Attempting to write empty matrix into cell state. +// lstm.Reset(); +// lstm.ResetCell(rho); +// arma::mat stepData(input.slice(0).memptr(), +// input.n_rows, input.n_cols, false, true); + +// lstm.Forward(stepData, // Input. +// outLstm, // Output. +// cellLstm, // Cell state. +// true); // Write into cell state. + +// for (size_t seqNum = 1; seqNum < rho; ++seqNum) +// { +// arma::mat empty; +// // Should throw error. +// REQUIRE_THROWS_AS(lstm.Forward(stepData, // Input. +// outLstm, // Output. +// empty, // Cell state. +// true), // Write into cell state. +// std::runtime_error); +// } +// } + +// /** +// * Test that the functions that can modify and access the parameters of the +// * GRU layer work. +// */ +// TEST_CASE("GRULayerParametersTest", "[ANNLayerTest]") +// { +// // Parameter order : inSize, outSize, rho. +// GRU<> layer1(1, 2, 3); +// GRU<> layer2(1, 2, 4); + +// // Make sure we can get the parameters successfully. +// REQUIRE(layer1.InSize() == 1); +// REQUIRE(layer1.OutSize() == 2); +// REQUIRE(layer1.Rho() == 3); + +// // Now modify the parameters to match the second layer. +// layer1.Rho() = 4; + +// // Now ensure all the results are the same. +// REQUIRE(layer1.InSize() == layer2.InSize()); +// REQUIRE(layer1.OutSize() == layer2.OutSize()); +// REQUIRE(layer1.Rho() == layer2.Rho()); +// } + +// /** +// * Check if the gradients computed by GRU cell are close enough to the +// * approximation of the gradients. +// */ +// TEST_CASE("GradientGRULayerTest", "[ANNLayerTest]") +// { +// // GRU function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(1, 1, 5)), +// target(arma::ones(1, 1, 5)) +// { +// const size_t rho = 5; + +// model = new RNN(rho); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(1, 10); +// model->Add >(10, 3, rho); +// model->Add >(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// arma::mat output; +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// RNN* model; +// arma::cube input, target; +// } function; + +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +// /** +// * GRU layer manual forward test. +// */ +// TEST_CASE("ForwardGRULayerTest", "[ANNLayerTest]") +// { +// // This will make it easier to clean memory later. +// GRU<>* gruAlloc = new GRU<>(3, 3, 5); +// GRU<>& gru = *gruAlloc; + +// // Initialize the weights to all ones. +// NetworkInitialization +// networkInit(ConstInitialization(1)); +// networkInit.Initialize(gru.Model(), gru.Parameters()); + +// // Provide input of all ones. +// arma::mat input = arma::ones(3, 1); +// arma::mat output; + +// gru.Forward(input, output); + +// // Compute the z_t gate output. +// arma::mat expectedOutput = arma::ones(3, 1); +// expectedOutput *= -4; +// expectedOutput = arma::exp(expectedOutput); +// expectedOutput = arma::ones(3, 1) / (arma::ones(3, 1) + expectedOutput); +// expectedOutput = (arma::ones(3, 1) - expectedOutput) % expectedOutput; + +// // For the first input the output should be equal to the output of +// // gate z_t as the previous output fed to the cell is all zeros. +// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); + +// expectedOutput = output; + +// gru.Forward(input, output); + +// double s = arma::as_scalar(arma::sum(expectedOutput)); + +// // Compute the value of z_t gate for the second input. +// arma::mat z_t = arma::ones(3, 1); +// z_t *= -(s + 4); +// z_t = arma::exp(z_t); +// z_t = arma::ones(3, 1) / (arma::ones(3, 1) + z_t); + +// // Compute the value of o_t gate for the second input. +// arma::mat o_t = arma::ones(3, 1); +// o_t *= -(arma::as_scalar(arma::sum(expectedOutput % z_t)) + 4); +// o_t = arma::exp(o_t); +// o_t = arma::ones(3, 1) / (arma::ones(3, 1) + o_t); + +// // Expected output for the second input. +// expectedOutput = z_t % expectedOutput + (arma::ones(3, 1) - z_t) % o_t; + +// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); + +// LayerTypes<> layer(gruAlloc); +// boost::apply_visitor(DeleteVisitor(), layer); +// } + +/** + * Simple add merge module test. + */ +// TEST_CASE("SimpleAddMergeLayerTest", "[ANNLayerTest]") +// { +// arma::mat output, input, delta; +// input = arma::ones(10, 1); +// +// for (size_t i = 0; i < 5; ++i) +// { +// AddMerge<> module(false, false); +// const size_t numMergeModules = RandInt(2, 10); +// for (size_t m = 0; m < numMergeModules; ++m) +// { +// IdentityLayer<> identityLayer; +// identityLayer.Forward(input, identityLayer.OutputParameter()); +// +// module.Add >(identityLayer); +// } +// +// // Test the Forward function. +// module.Forward(input, output); +// REQUIRE(10 * numMergeModules == arma::accu(output)); +// +// // Test the Backward function. +// module.Backward(input, output, delta); +// REQUIRE(arma::accu(output) == arma::accu(delta)); +// } +// } + +/** + * Test the LSTM layer with a user defined rho parameter and without. + */ +// TEST_CASE("LSTMRrhoTest", "[ANNLayerTest]") +// { +// const size_t rho = 5; +// arma::cube input = arma::randu(1, 1, 5); +// arma::cube target = arma::zeros(1, 1, 5); +// RandomInitialization init(0.5, 0.5); +// +// // Create model with user defined rho parameter. +// RNN modelA( +// rho, false, NegativeLogLikelihood(), init); +// modelA.Add >(); +// modelA.Add >(1, 10); +// +// // Use LSTM layer with rho. +// modelA.Add >(10, 3, rho); +// modelA.Add >(); +// +// // Create model without user defined rho parameter. +// RNN modelB( +// rho, false, NegativeLogLikelihood(), init); +// modelB.Add >(); +// modelB.Add >(1, 10); +// +// // Use LSTM layer with rho = MAXSIZE. +// modelB.Add >(10, 3); +// modelB.Add >(); +// +// ens::StandardSGD opt(0.1, 1, 5, -100, false); +// modelA.Train(input, target, opt); +// modelB.Train(input, target, opt); +// +// CheckMatrices(modelB.Parameters(), modelA.Parameters()); +// } + +/** + * LSTM layer numerical gradient test. + */ +// TEST_CASE("GradientLSTMLayerTest", "[ANNLayerTest]") +// { +// // LSTM function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(1, 1, 5)), +// target(arma::zeros(1, 1, 5)) +// { +// const size_t rho = 5; +// +// model = new RNN(rho); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(1, 10); +// model->Add >(10, 3, rho); +// model->Add >(); +// } +// +// ~GradientFunction() +// { +// delete model; +// } +// +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } +// +// arma::mat& Parameters() { return model->Parameters(); } +// +// RNN* model; +// arma::cube input, target; +// } function; +// +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +/** + * Test that the functions that can modify and access the parameters of the + * LSTM layer work. + */ +// TEST_CASE("LSTMLayerParametersTest", "[ANNLayerTest]") +// { +// // Parameter order : inSize, outSize, rho. +// LSTM<> layer1(1, 2, 3); +// LSTM<> layer2(1, 2, 4); +// +// // Make sure we can get the parameters successfully. +// REQUIRE(layer1.InSize() == 1); +// REQUIRE(layer1.OutSize() == 2); +// REQUIRE(layer1.Rho() == 3); +// +// // Now modify the parameters to match the second layer. +// layer1.Rho() = 4; +// +// // Now ensure all the results are the same. +// REQUIRE(layer1.InSize() == layer2.InSize()); +// REQUIRE(layer1.OutSize() == layer2.OutSize()); +// REQUIRE(layer1.Rho() == layer2.Rho()); +// } + +/** + * Test the FastLSTM layer with a user defined rho parameter and without. + */ +// TEST_CASE("FastLSTMRrhoTest", "[ANNLayerTest]") +// { +// const size_t rho = 5; +// arma::cube input = arma::randu(1, 1, 5); +// arma::cube target = arma::zeros(1, 1, 5); +// RandomInitialization init(0.5, 0.5); +// +// // Create model with user defined rho parameter. +// RNN modelA( +// rho, false, NegativeLogLikelihood(), init); +// modelA.Add >(); +// modelA.Add >(1, 10); +// +// // Use FastLSTM layer with rho. +// modelA.Add >(10, 3, rho); +// modelA.Add >(); +// +// // Create model without user defined rho parameter. +// RNN modelB( +// rho, false, NegativeLogLikelihood(), init); +// modelB.Add >(); +// modelB.Add >(1, 10); +// +// // Use FastLSTM layer with rho = MAXSIZE. +// modelB.Add >(10, 3); +// modelB.Add >(); +// +// ens::StandardSGD opt(0.1, 1, 5, -100, false); +// modelA.Train(input, target, opt); +// modelB.Train(input, target, opt); +// +// CheckMatrices(modelB.Parameters(), modelA.Parameters()); +// } + +/** + * FastLSTM layer numerical gradient test. + */ +// TEST_CASE("GradientFastLSTMLayerTest", "[ANNLayerTest]") +// { +// // Fast LSTM function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(1, 1, 5)), +// target(arma::zeros(1, 1, 5)) +// { +// const size_t rho = 5; +// +// model = new RNN(rho); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(1, 10); +// model->Add >(10, 3, rho); +// model->Add >(); +// } +// +// ~GradientFunction() +// { +// delete model; +// } +// +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } +// +// arma::mat& Parameters() { return model->Parameters(); } +// +// RNN* model; +// arma::cube input, target; +// } function; +// +// // The threshold should be << 0.1 but since the Fast LSTM layer uses an +// // approximation of the sigmoid function the estimated gradient is not +// // correct. +// REQUIRE(CheckGradient(function) <= 0.2); +// } + +/** + * Test that the functions that can modify and access the parameters of the + * Fast LSTM layer work. + */ +// TEST_CASE("FastLSTMLayerParametersTest", "[ANNLayerTest]") +// { +// // Parameter order : inSize, outSize, rho. +// FastLSTM<> layer1(1, 2, 3); +// FastLSTM<> layer2(1, 2, 4); +// +// // Make sure we can get the parameters successfully. +// REQUIRE(layer1.InSize() == 1); +// REQUIRE(layer1.OutSize() == 2); +// REQUIRE(layer1.Rho() == 3); +// +// // Now modify the parameters to match the second layer. +// layer1.Rho() = 4; +// +// // Now ensure all the results are the same. +// REQUIRE(layer1.InSize() == layer2.InSize()); +// REQUIRE(layer1.OutSize() == layer2.OutSize()); +// REQUIRE(layer1.Rho() == layer2.Rho()); +// } + +/** + * Check whether copying and moving network with FastLSTM is working or not. + */ +// TEST_CASE("CheckCopyMoveFastLSTMTest", "[ANNLayerTest]") +// { +// arma::cube input = arma::randu(1, 1, 5); +// arma::cube target = arma::ones(1, 1, 5); +// const size_t rho = 5; +// +// RNN *model1 = +// new RNN(rho); +// model1->ResetData(input, target); +// model1->Add >(); +// model1->Add >(1, 10); +// model1->Add >(10, 3, rho); +// model1->Add >(); +// +// RNN *model2 = +// new RNN(rho); +// model2->ResetData(input, target); +// model2->Add >(); +// model2->Add >(1, 10); +// model2->Add >(10, 3, rho); +// model2->Add >(); +// +// // Check whether copy constructor is working or not. +// CheckRNNCopyFunction<>(model1, input, target, 1); +// +// // Check whether move constructor is working or not. +// CheckRNNMoveFunction<>(model2, input, target, 1); +// } + +/** + * Check whether copying and moving network with LSTM is working or not. + */ +// TEST_CASE("CheckCopyMoveLSTMTest", "[ANNLayerTest]") +// { +// arma::cube input = arma::randu(1, 1, 5); +// arma::cube target = arma::ones(1, 1, 5); +// const size_t rho = 5; +// +// RNN *model1 = +// new RNN(rho); +// model1->ResetData(input, target); +// model1->Add >(); +// model1->Add >(1, 10); +// model1->Add >(10, 3, rho); +// model1->Add >(); +// +// RNN *model2 = +// new RNN(rho); +// model2->ResetData(input, target); +// model2->Add >(); +// model2->Add >(1, 10); +// model2->Add >(10, 3, rho); +// model2->Add >(); +// +// // Check whether copy constructor is working or not. +// CheckRNNCopyFunction<>(model1, input, target, 1); +// +// // Check whether move constructor is working or not. +// CheckRNNMoveFunction<>(model2, input, target, 1); +// } + +/** + * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell + * state. Besides output, the overloaded function provides read access to cell + * state of the LSTM layer. + */ +// TEST_CASE("ReadCellStateParamLSTMLayerTest", "[ANNLayerTest]") +// { +// const size_t rho = 5, inputSize = 3, outputSize = 2; +// +// // Provide input of all ones. +// arma::cube input = arma::ones(inputSize, outputSize, rho); +// +// arma::mat inputGate, forgetGate, outputGate, hidden; +// arma::mat outLstm, cellLstm; +// +// // LSTM layer. +// LSTM<> lstm(inputSize, outputSize, rho); +// lstm.Reset(); +// lstm.ResetCell(rho); +// +// // Initialize the weights to all ones. +// lstm.Parameters().ones(); +// +// arma::mat inputWeight = arma::ones(outputSize, inputSize); +// arma::mat outputWeight = arma::ones(outputSize, outputSize); +// arma::mat bias = arma::ones(outputSize, input.n_cols); +// arma::mat cellCalc = arma::zeros(outputSize, input.n_cols); +// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); +// +// for (size_t seqNum = 0; seqNum < rho; ++seqNum) +// { +// // Wrap a matrix around our data to avoid a copy. +// arma::mat stepData(input.slice(seqNum).memptr(), +// input.n_rows, input.n_cols, false, true); +// +// // Apply Forward() on LSTM layer. +// lstm.Forward(stepData, // Input. +// outLstm, // Output. +// cellLstm, // Cell state. +// false); // Don't write into the cell state. +// +// // Compute the value of cell state and output. +// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); +// +// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); +// +// // z = tanh(W.dot(x) + W.dot(h) + b). +// hidden = arma::tanh(inputWeight * stepData + +// outputWeight * outCalc + bias); +// +// // c = f * c + i * z. +// cellCalc = forgetGate % cellCalc + inputGate % hidden; +// +// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); +// +// // h = o * tanh(c). +// outCalc = outputGate % arma::tanh(cellCalc); +// +// CheckMatrices(outLstm, outCalc, 1e-12); +// CheckMatrices(cellLstm, cellCalc, 1e-12); +// } +// } + +/** + * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell + * state. Besides output, the overloaded function provides write access to cell + * state of the LSTM layer. + */ +// TEST_CASE("WriteCellStateParamLSTMLayerTest", "[ANNLayerTest]") +// { +// const size_t rho = 5, inputSize = 3, outputSize = 2; +// +// // Provide input of all ones. +// arma::cube input = arma::ones(inputSize, outputSize, rho); +// +// arma::mat inputGate, forgetGate, outputGate, hidden; +// arma::mat outLstm, cellLstm; +// arma::mat cellCalc; +// +// // LSTM layer. +// LSTM<> lstm(inputSize, outputSize, rho); +// lstm.Reset(); +// lstm.ResetCell(rho); +// +// // Initialize the weights to all ones. +// lstm.Parameters().ones(); +// +// arma::mat inputWeight = arma::ones(outputSize, inputSize); +// arma::mat outputWeight = arma::ones(outputSize, outputSize); +// arma::mat bias = arma::ones(outputSize, input.n_cols); +// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); +// +// for (size_t seqNum = 0; seqNum < rho; ++seqNum) +// { +// // Wrap a matrix around our data to avoid a copy. +// arma::mat stepData(input.slice(seqNum).memptr(), +// input.n_rows, input.n_cols, false, true); +// +// if (cellLstm.is_empty()) +// { +// // Set the cell state to zeros. +// cellLstm = arma::zeros(outputSize, input.n_cols); +// cellCalc = arma::zeros(outputSize, input.n_cols); +// } +// else +// { +// // Set the cell state to zeros. +// cellLstm = arma::zeros(cellLstm.n_rows, cellLstm.n_cols); +// cellCalc = arma::zeros(cellCalc.n_rows, cellCalc.n_cols); +// } +// +// // Apply Forward() on the LSTM layer. +// lstm.Forward(stepData, // Input. +// outLstm, // Output. +// cellLstm, // Cell state. +// true); // Write into cell state. +// +// // Compute the value of cell state and output. +// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); +// +// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); +// +// // z = tanh(W.dot(x) + W.dot(h) + b). +// hidden = arma::tanh(inputWeight * stepData + +// outputWeight * outCalc + bias); +// +// // c = f * c + i * z. +// cellCalc = forgetGate % cellCalc + inputGate % hidden; +// +// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); +// +// // h = o * tanh(c). +// outCalc = outputGate % arma::tanh(cellCalc); +// +// CheckMatrices(outLstm, outCalc, 1e-12); +// CheckMatrices(cellLstm, cellCalc, 1e-12); +// } +// +// // Attempting to write empty matrix into cell state. +// lstm.Reset(); +// lstm.ResetCell(rho); +// arma::mat stepData(input.slice(0).memptr(), +// input.n_rows, input.n_cols, false, true); +// +// lstm.Forward(stepData, // Input. +// outLstm, // Output. +// cellLstm, // Cell state. +// true); // Write into cell state. +// +// for (size_t seqNum = 1; seqNum < rho; ++seqNum) +// { +// arma::mat empty; +// // Should throw error. +// REQUIRE_THROWS_AS(lstm.Forward(stepData, // Input. +// outLstm, // Output. +// empty, // Cell state. +// true), // Write into cell state. +// std::runtime_error); +// } +// } + +/** + * Test that the functions that can modify and access the parameters of the + * GRU layer work. + */ +// TEST_CASE("GRULayerParametersTest", "[ANNLayerTest]") +// { +// // Parameter order : inSize, outSize, rho. +// GRU<> layer1(1, 2, 3); +// GRU<> layer2(1, 2, 4); +// +// // Make sure we can get the parameters successfully. +// REQUIRE(layer1.InSize() == 1); +// REQUIRE(layer1.OutSize() == 2); +// REQUIRE(layer1.Rho() == 3); +// +// // Now modify the parameters to match the second layer. +// layer1.Rho() = 4; +// +// // Now ensure all the results are the same. +// REQUIRE(layer1.InSize() == layer2.InSize()); +// REQUIRE(layer1.OutSize() == layer2.OutSize()); +// REQUIRE(layer1.Rho() == layer2.Rho()); +// } + +/** + * Check if the gradients computed by GRU cell are close enough to the + * approximation of the gradients. + */ +// TEST_CASE("GradientGRULayerTest", "[ANNLayerTest]") +// { +// // GRU function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(1, 1, 5)), +// target(arma::zeros(1, 1, 5)) +// { +// const size_t rho = 5; +// +// model = new RNN(rho); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(1, 10); +// model->Add >(10, 3, rho); +// model->Add >(); +// } +// +// ~GradientFunction() +// { +// delete model; +// } +// +// double Gradient(arma::mat& gradient) const +// { +// arma::mat output; +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } +// +// arma::mat& Parameters() { return model->Parameters(); } +// +// RNN* model; +// arma::cube input, target; +// } function; +// +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +/** + * GRU layer manual forward test. + */ +// TEST_CASE("ForwardGRULayerTest", "[ANNLayerTest]") +// { +// // This will make it easier to clean memory later. +// GRU<>* gruAlloc = new GRU<>(3, 3, 5); +// GRU<>& gru = *gruAlloc; +// +// // Initialize the weights to all ones. +// NetworkInitialization +// networkInit(ConstInitialization(1)); +// networkInit.Initialize(gru.Model(), gru.Parameters()); +// +// // Provide input of all ones. +// arma::mat input = arma::ones(3, 1); +// arma::mat output; +// +// gru.Forward(input, output); +// +// // Compute the z_t gate output. +// arma::mat expectedOutput = arma::ones(3, 1); +// expectedOutput *= -4; +// expectedOutput = arma::exp(expectedOutput); +// expectedOutput = arma::ones(3, 1) / (arma::ones(3, 1) + expectedOutput); +// expectedOutput = (arma::ones(3, 1) - expectedOutput) % expectedOutput; +// +// // For the first input the output should be equal to the output of +// // gate z_t as the previous output fed to the cell is all zeros. +// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); +// +// expectedOutput = output; +// +// gru.Forward(input, output); +// +// double s = arma::as_scalar(arma::sum(expectedOutput)); +// +// // Compute the value of z_t gate for the second input. +// arma::mat z_t = arma::ones(3, 1); +// z_t *= -(s + 4); +// z_t = arma::exp(z_t); +// z_t = arma::ones(3, 1) / (arma::ones(3, 1) + z_t); +// +// // Compute the value of o_t gate for the second input. +// arma::mat o_t = arma::ones(3, 1); +// o_t *= -(arma::as_scalar(arma::sum(expectedOutput % z_t)) + 4); +// o_t = arma::exp(o_t); +// o_t = arma::ones(3, 1) / (arma::ones(3, 1) + o_t); +// +// // Expected output for the second input. +// expectedOutput = z_t % expectedOutput + (arma::ones(3, 1) - z_t) % o_t; +// +// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); +// +// LayerTypes<> layer(gruAlloc); +// boost::apply_visitor(DeleteVisitor(), layer); +// } + +/** + * Simple lookup module test. + * +TEST_CASE("SimpleLookupLayerTest", "[ANNLayerTest]") +{ + const size_t vocabSize = 10; + const size_t embeddingSize = 2; + const size_t seqLength = 3; + const size_t batchSize = 4; + + arma::mat output, input, gy, g, gradient; + + Lookup module(vocabSize, embeddingSize); + module.Parameters().randu(); + + // Test the Forward function. + input = arma::zeros(seqLength, batchSize); + for (size_t i = 0; i < input.n_elem; ++i) + { + int token = RandInt(1, vocabSize); + input(i) = token; + } + + module.Forward(input, output); + for (size_t i = 0; i < batchSize; ++i) + { + // The Lookup module uses index - 1 for the cols. + const double outputSum = arma::accu(module.Parameters().cols( + arma::conv_to::from(input.col(i)) - 1)); + + REQUIRE(std::fabs(outputSum - arma::accu(output.col(i))) <= 1e-5); + } + + // Test the Gradient function. + arma::mat error = 0.01 * arma::randu(embeddingSize * seqLength, batchSize); + module.Gradient(input, error, gradient); + + REQUIRE(std::fabs(arma::accu(error) - arma::accu(gradient)) <= 1e-07); +} +*/ + +/** + * Lookup layer numerical gradient test. + * +TEST_CASE("GradientLookupLayerTest", "[ANNLayerTest]") +{ + // Lookup function gradient instantiation. + struct GradientFunction + { + GradientFunction() + { + input.set_size(seqLength, batchSize); + for (size_t i = 0; i < input.n_elem; ++i) + { + input(i) = RandInt(1, vocabSize); + } + target = arma::zeros(vocabSize, batchSize); + for (size_t i = 0; i < batchSize; ++i) + { + const size_t targetWord = RandInt(1, vocabSize); + target(targetWord, i) = 1; + } + + model = new FFN, GlorotInitialization>(BCELoss<>(1e-10, false)); + model->ResetData(input, target); + model->Add(vocabSize, embeddingSize); + model->Add(embeddingSize * seqLength, vocabSize); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, batchSize); + model->Gradient(model->Parameters(), 0, gradient, batchSize); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN, GlorotInitialization>* model; + arma::mat input, target; + + const size_t seqLength = 10; + const size_t embeddingSize = 8; + const size_t vocabSize = 20; + const size_t batchSize = 4; + } function; + + REQUIRE(CheckGradient(function) <= 1e-6); +} +*/ + +/** + * Test that the functions that can access the parameters of the + * Lookup layer work. + * +TEST_CASE("LookupLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : vocabSize, embedingSize. + Lookup layer(100, 8); + + // Make sure we can get the parameters successfully. + REQUIRE(layer.VocabSize() == 100); + REQUIRE(layer.EmbeddingSize() == 8); +} +*/ + +/** + * Simple test for the NearestInterpolation layer + * +TEST_CASE("SimpleNearestInterpolationLayerTest", "[ANNLayerTest]") +{ + // Tested output against torch.nn.Upsample(mode="nearest"). + arma::mat input, output, unzoomedOutput, expectedOutput; + size_t inRowSize = 2; + size_t inColSize = 2; + size_t outRowSize = 5; + size_t outColSize = 7; + size_t depth = 1; + input.zeros(inRowSize * inColSize * depth, 1); + input[0] = 1.0; + input[1] = 3.0; + input[2] = 2.0; + input[3] = 4.0; + NearestInterpolation<> layer(inRowSize, inColSize, outRowSize, + outColSize, depth); + + expectedOutput << 1.0000 << 1.0000 << 1.0000 << 1.0000 << 2.0000 + << 2.0000 << 2.0000 << arma::endr + << 1.0000 << 1.0000 << 1.0000 << 1.0000 << 2.0000 + << 2.0000 << 2.0000 << arma::endr + << 1.0000 << 1.0000 << 1.0000 << 1.0000 << 2.0000 + << 2.0000 << 2.0000 << arma::endr + << 3.0000 << 3.0000 << 3.0000 << 3.0000 << 4.0000 + << 4.0000 << 4.0000 << arma::endr + << 3.0000 << 3.0000 << 3.0000 << 3.0000 << 4.0000 + << 4.0000 << 4.0000 << arma::endr; + expectedOutput.reshape(35, 1); + + layer.Forward(input, output); + CheckMatrices(output - expectedOutput, + arma::zeros(output.n_rows), 1e-4); + + expectedOutput.clear(); + expectedOutput << 12.0000 << 18.0000 << arma::endr + << 24.0000 << 24.0000 << arma::endr; + expectedOutput.reshape(4, 1); + layer.Backward(output, output, unzoomedOutput); + CheckMatrices(unzoomedOutput - expectedOutput, + arma::zeros(input.n_rows), 1e-4); + + arma::mat input1, output1, unzoomedOutput1, expectedOutput1; + inRowSize = 2; + inColSize = 3; + outRowSize = 17; + outColSize = 23; + input1 << 1 << 2 << 3 << arma::endr + << 4 << 5 << 6 << arma::endr; + input1.reshape(6, 1); + NearestInterpolation<> layer1(inRowSize, inColSize, outRowSize, + outColSize, depth); + + layer1.Forward(input1, output1); + layer1.Backward(output1, output1, unzoomedOutput1); + + REQUIRE(arma::accu(output1) - 1317.00 == Approx(0.0).margin(1e-05)); + REQUIRE(arma::accu(unzoomedOutput1) - 1317.00 == + Approx(0.0).margin(1e-05)); +} +*/ + +/* + * Simple test for the BilinearInterpolation layer + * +TEST_CASE("SimpleBilinearInterpolationLayerTest", "[ANNLayerTest]") +{ + // Tested output against tensorflow.image.resize_bilinear() + arma::mat input, output, unzoomedOutput, expectedOutput; + size_t inRowSize = 2; + size_t inColSize = 2; + size_t outRowSize = 5; + size_t outColSize = 5; + size_t depth = 1; + input.zeros(inRowSize * inColSize * depth, 1); + input[0] = 1.0; + input[1] = input[2] = 2.0; + input[3] = 3.0; + BilinearInterpolation layer(inRowSize, inColSize, outRowSize, outColSize, + depth); + expectedOutput = arma::mat("1.0000 1.4000 1.8000 2.0000 2.0000 \ + 1.4000 1.8000 2.2000 2.4000 2.4000 \ + 1.8000 2.2000 2.6000 2.8000 2.8000 \ + 2.0000 2.4000 2.8000 3.0000 3.0000 \ + 2.0000 2.4000 2.8000 3.0000 3.0000"); + expectedOutput.reshape(25, 1); + layer.Forward(input, output); + CheckMatrices(output - expectedOutput, arma::zeros(output.n_rows), 1e-12); + + expectedOutput = arma::mat("1.0000 1.9000 1.9000 2.8000"); + expectedOutput.reshape(4, 1); + layer.Backward(output, output, unzoomedOutput); + CheckMatrices(unzoomedOutput - expectedOutput, + arma::zeros(input.n_rows), 1e-12); +} +*/ + +/** + * Test that the functions that can modify and access the parameters of the + * Bilinear Interpolation layer work. + * +TEST_CASE("BilinearInterpolationLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : inRowSize, inColSize, outRowSize, outColSize, depth. + BilinearInterpolation layer1(1, 2, 3, 4, 5); + BilinearInterpolation layer2(2, 3, 4, 5, 6); + + // Make sure we can get the parameters successfully. + REQUIRE(layer1.InRowSize() == 1); + REQUIRE(layer1.InColSize() == 2); + REQUIRE(layer1.OutRowSize() == 3); + REQUIRE(layer1.OutColSize() == 4); + REQUIRE(layer1.InDepth() == 5); + + // Now modify the parameters to match the second layer. + layer1.InRowSize() = 2; + layer1.InColSize() = 3; + layer1.OutRowSize() = 4; + layer1.OutColSize() = 5; + layer1.InDepth() = 6; + + // Now ensure all results are the same. + REQUIRE(layer1.InRowSize() == layer2.InRowSize()); + REQUIRE(layer1.InColSize() == layer2.InColSize()); + REQUIRE(layer1.OutRowSize() == layer2.OutRowSize()); + REQUIRE(layer1.OutColSize() == layer2.OutColSize()); + REQUIRE(layer1.InDepth() == layer2.InDepth()); +} +*/ + +/* + * Simple test for the BicubicInterpolation layer. + * +TEST_CASE("SimpleBicubicInterpolationLayerTest", "[ANNLayerTest]") +{ + // Tested output against torch.nn.Upsample(mode="bicubic"). + // Test case with square input with rectangular output. + arma::mat input, output, unzoomedOutput, expectedOutput; + size_t inRowSize = 2; + size_t inColSize = 2; + size_t outRowSize = 5; + size_t outColSize = 7; + size_t depth = 1; + input.zeros(inRowSize * inColSize * depth, 1); + + input << 10 << 20 << arma::endr + << 30 << 40 << arma::endr; + input.reshape(4, 1); + BicubicInterpolation<> layer(inRowSize, inColSize, outRowSize, + outColSize, depth); + + expectedOutput << 6.68803935860 << 7.33308309038 << 9.69733236152 + << 12.79500000000 << 15.89266763848 << 18.25691690962 + << 18.90196064140 << arma::endr + << 10.53303935860 << 11.17808309038 << 13.54233236152 + << 16.64000000000 << 19.73766763848 << 22.10191690962 + << 22.74696064140 << arma::endr + << 18.89303935860 << 19.53808309038 << 21.90233236152 + << 25.00000000000 << 28.09766763848 << 30.46191690962 + << 31.10696064140 << arma::endr + << 27.25303935860 << 27.89808309038 << 30.26233236152 + << 33.36000000000 << 36.45766763848 << 38.82191690962 + << 39.46696064140 << arma::endr + << 31.09803935860 << 31.74308309038 << 34.10733236152 + << 37.20500000000 << 40.30266763848 << 42.66691690962 + << 43.31196064140 << arma::endr; + expectedOutput.reshape(35, 1); + layer.Forward(input, output); + + CheckMatrices(output, expectedOutput, 1e-6); + + expectedOutput.clear(); + expectedOutput << 103.79040654914 << 180.51345595086 << arma::endr + << 256.98654404914 << 333.70959345086 << arma::endr; + expectedOutput.reshape(4, 1); + + layer.Backward(output, output, unzoomedOutput); + + CheckMatrices(unzoomedOutput, expectedOutput, 1e-6); + + // Tested output against torch.nn.Upsample(mode="bicubic"). + // Test case with rectangular input with rectangular output. + arma::mat input1, output1, unzoomedOutput1, expectedOutput1, expectedUnzoomed; + + inRowSize = 2; + inColSize = 3; + outRowSize = 5; + outColSize = 7; + depth = 1; + input1.zeros(inRowSize * inColSize * depth, 1); + + input1 << 10 << 20 << 30 << arma::endr + << 40 << 50 << 60 << arma::endr; + input1.reshape(6, 1); + + BicubicInterpolation<> layer1(inRowSize, inColSize, outRowSize, + outColSize, depth); + + expectedOutput1 << 5.59920553936 << 7.77121720117 << 11.44468658892 + << 16.69250000000 << 21.94031341108 << 25.61378279883 + << 27.78579446064 << arma::endr + << 11.36670553936 << 13.53871720117 << 17.21218658892 + << 22.46000000000 << 27.70781341108 << 31.38128279883 + << 33.55329446064 << arma::endr + << 23.90670553936 << 26.07871720117 << 29.75218658892 + << 35.00000000000 << 40.24781341108 << 43.92128279883 + << 46.09329446064 << arma::endr + << 36.44670553936 << 38.61871720117 << 42.29218658892 + << 47.54000000000 << 52.78781341108 << 56.46128279883 + << 58.63329446064 << arma::endr + << 42.21420553936 << 44.38621720117 << 48.05968658892 + << 53.30750000000 << 58.55531341108 << 62.22878279883 + << 64.40079446064 << arma::endr; + expectedOutput1.reshape(35, 1); + layer1.Forward(input1, output1); + + CheckMatrices(output1, expectedOutput1, 1e-6); + + expectedUnzoomed << 67.65674505130 << 132.29729646501 + << 182.75175223368 << arma::endr + << 218.01355388877 << 291.17209129009 + << 333.10856107115 << arma::endr; + expectedUnzoomed.reshape(6, 1); + + layer1.Backward(output1, output1, unzoomedOutput1); + CheckMatrices(unzoomedOutput1, expectedUnzoomed, 1e-6); +} +*/ + +/** + * VirtualBatchNorm layer numerical gradient test. + * +TEST_CASE("GradientVirtualBatchNormTest", "[ANNLayerTest]") +{ + // Add function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randn(5, 256)), + target(arma::zeros(1, 256)) + { + arma::mat referenceBatch = arma::mat(input.memptr(), input.n_rows, 4); + + model = new FFN(); + model->ResetData(input, target); + model->Add(); + model->Add(5, 5); + model->Add(referenceBatch, 5); + model->Add(5, 2); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 16, false); + model->Gradient(model->Parameters(), 0, gradient, 16); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + REQUIRE(CheckGradient(function) <= 1e-4); +} +*/ + +/** + * Test that the functions that can modify and access the parameters of the + * Virtual Batch Norm layer work. + * +TEST_CASE("VirtualBatchNormLayerParametersTest", "[ANNLayerTest]") +{ + arma::mat input = arma::randn(5, 16); + arma::mat referenceBatch = arma::mat(input.memptr(), input.n_rows, 4); + + // Parameter order : referenceBatch, size, eps. + VirtualBatchNorm layer(referenceBatch, 5, 1e-3); + + // Make sure we can get the parameters successfully. + REQUIRE(layer.InSize() == 5); + REQUIRE(layer.Epsilon() == 1e-3); +} +*/ + +// /** +// * MiniBatchDiscrimination layer numerical gradient test. +// */ +// TEST_CASE("MiniBatchDiscriminationTest", "[ANNLayerTest]") +// { +// // Add function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randn(5, 4)), +// target(arma::zeros(1, 4)) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(5, 5); +// model->Add >(5, 10, 16); +// model->Add >(10, 2); +// model->Add >(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// return model->EvaluateWithGradient(model->Parameters(), 0, gradient, 4); +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// FFN* model; +// arma::mat input, target; +// } function; + +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +/** + * Simple Transposed Convolution layer test. + * +TEST_CASE("SimpleTransposedConvolutionLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + + TransposedConvolution module1(1, 1, 3, 3, 1, 1, 0, 0, 4, 4, 6, 6); + // Test the forward function. + input = arma::linspace(0, 15, 16); + module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module1.Parameters()(0) = 1.0; + module1.Parameters()(8) = 2.0; + module1.Reset(); + module1.Forward(input, output); + // Value calculated using tensorflow.nn.conv2d_transpose() + REQUIRE(arma::accu(output) == 360.0); + + // Test the backward function. + module1.Backward(input, output, delta); + // Value calculated using tensorflow.nn.conv2d() + REQUIRE(arma::accu(delta) == 720.0); + + TransposedConvolution module2(1, 1, 4, 4, 1, 1, 1, 1, 5, 5, 6, 6); + // Test the forward function. + input = arma::linspace(0, 24, 25); + module2.Parameters() = arma::mat(16 + 1, 1, arma::fill::zeros); + module2.Parameters()(0) = 1.0; + module2.Parameters()(3) = 1.0; + module2.Parameters()(6) = 1.0; + module2.Parameters()(9) = 1.0; + module2.Parameters()(12) = 1.0; + module2.Parameters()(15) = 2.0; + module2.Reset(); + module2.Forward(input, output); + // Value calculated using torch.nn.functional.conv_transpose2d() + REQUIRE(arma::accu(output) == 1512.0); + + // Test the backward function. + module2.Backward(input, output, delta); + // Value calculated using torch.nn.functional.conv2d() + REQUIRE(arma::accu(delta) == 6504.0); + + TransposedConvolution module3(1, 1, 3, 3, 1, 1, 1, 1, 5, 5, 5, 5); + // Test the forward function. + input = arma::linspace(0, 24, 25); + module3.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module3.Parameters()(1) = 2.0; + module3.Parameters()(2) = 4.0; + module3.Parameters()(3) = 3.0; + module3.Parameters()(8) = 1.0; + module3.Reset(); + module3.Forward(input, output); + // Value calculated using torch.nn.functional.conv_transpose2d() + REQUIRE(arma::accu(output) == 2370.0); + + // Test the backward function. + module3.Backward(input, output, delta); + // Value calculated using torch.nn.functional.conv2d() + REQUIRE(arma::accu(delta) == 19154.0); + + TransposedConvolution module4(1, 1, 3, 3, 1, 1, 0, 0, 5, 5, 7, 7); + // Test the forward function. + input = arma::linspace(0, 24, 25); + module4.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module4.Parameters()(2) = 2.0; + module4.Parameters()(4) = 4.0; + module4.Parameters()(6) = 6.0; + module4.Parameters()(8) = 8.0; + module4.Reset(); + module4.Forward(input, output); + // Value calculated using torch.nn.functional.conv_transpose2d() + REQUIRE(arma::accu(output) == 6000.0); + + // Test the backward function. + module4.Backward(input, output, delta); + // Value calculated using torch.nn.functional.conv2d() + REQUIRE(arma::accu(delta) == 86208.0); + + TransposedConvolution module5(1, 1, 3, 3, 2, 2, 0, 0, 2, 2, 5, 5); + // Test the forward function. + input = arma::linspace(0, 3, 4); + module5.Parameters() = arma::mat(25 + 1, 1, arma::fill::zeros); + module5.Parameters()(2) = 8.0; + module5.Parameters()(4) = 6.0; + module5.Parameters()(6) = 4.0; + module5.Parameters()(8) = 2.0; + module5.Reset(); + module5.Forward(input, output); + // Value calculated using torch.nn.functional.conv_transpose2d() + REQUIRE(arma::accu(output) == 120.0); + + // Test the backward function. + module5.Backward(input, output, delta); + // Value calculated using torch.nn.functional.conv2d() + REQUIRE(arma::accu(delta) == 960.0); + + TransposedConvolution module6(1, 1, 3, 3, 2, 2, 1, 1, 3, 3, 5, 5); + // Test the forward function. + input = arma::linspace(0, 8, 9); + module6.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module6.Parameters()(0) = 8.0; + module6.Parameters()(3) = 6.0; + module6.Parameters()(6) = 2.0; + module6.Parameters()(8) = 4.0; + module6.Reset(); + module6.Forward(input, output); + // Value calculated using torch.nn.functional.conv_transpose2d() + REQUIRE(arma::accu(output) == 410.0); + + // Test the backward function. + module6.Backward(input, output, delta); + // Value calculated using torch.nn.functional.conv2d() + REQUIRE(arma::accu(delta) == 4444.0); + + TransposedConvolution module7(1, 1, 3, 3, 2, 2, 1, 1, 3, 3, 6, 6); + // Test the forward function. + input = arma::linspace(0, 8, 9); + module7.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module7.Parameters()(0) = 8.0; + module7.Parameters()(2) = 6.0; + module7.Parameters()(4) = 2.0; + module7.Parameters()(8) = 4.0; + module7.Reset(); + module7.Forward(input, output); + // Value calculated using torch.nn.functional.conv_transpose2d() + REQUIRE(arma::accu(output) == 606.0); + + module7.Backward(input, output, delta); + // Value calculated using torch.nn.functional.conv2d() + REQUIRE(arma::accu(delta) == 7732.0); +} +*/ + +/** + * Transposed Convolution layer numerical gradient test. + * +TEST_CASE("GradientTransposedConvolutionLayerTest", "[ANNLayerTest]") +{ + // Add function gradient instantiation. + // To make this test robust, check it five times. + bool pass = false; + for (size_t trial = 0; trial < 5; trial++) + { + struct GradientFunction + { + GradientFunction() : + input(arma::linspace(0, 35, 36)), + target(arma::mat("0")) + { + model = new FFN(); + model->ResetData(input, target); + model->Add(1, 1, 3, 3, 2, 2, 1, 1, 6, 6, 12, 12); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + if (CheckGradient(function) < 1e-3) + { + pass = true; + break; + } + } + REQUIRE(pass == true); +} +*/ + +/** + * Simple MultiplyMerge module test. + * +TEST_CASE("SimpleMultiplyMergeLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + input = arma::ones(10, 1); + + for (size_t i = 0; i < 5; ++i) + { + MultiplyMerge module(false, false); + const size_t numMergeModules = RandInt(2, 10); + for (size_t m = 0; m < numMergeModules; ++m) + { + IdentityLayer* identityLayer = new IdentityLayer(); + identityLayer->Forward(input, identityLayer->OutputParameter()); + + module.Add(identityLayer); + } + + // Test the Forward function. + module.Forward(input, output); + REQUIRE(10 == arma::accu(output)); + + // Test the Backward function. + module.Backward(input, output, delta); + REQUIRE(arma::accu(output) == arma::accu(delta)); + } +} +*/ + +/** + * Check whether copying and moving network with MultiplyMerge is working or + * not. + */ +// TEST_CASE("CheckCopyMoveMultiplyMergeTest", "[ANNLayerTest]") +// { +// arma::mat input(10, 1); +// input.randu(); +// +// arma::mat output1; +// arma::mat output2; +// arma::mat output3; +// arma::mat output4; +// +// const size_t numMergeModules = RandInt(2, 10); +// +// MultiplyMerge<> *module1 = new MultiplyMerge<>(true, false); +// for (size_t m = 0; m < numMergeModules; ++m) +// { +// IdentityLayer<> identityLayer; +// identityLayer.Forward(input, identityLayer.OutputParameter()); +// +// module1->Add >(identityLayer); +// } +// +// module1->Forward(input, output1); +// +// MultiplyMerge<> module2 = *module1; +// delete module1; +// +// module2.Forward(input, output2); +// CheckMatrices(output1, output2); +// +// MultiplyMerge<> *module3 = new MultiplyMerge<>(true, false); +// for (size_t m = 0; m < numMergeModules; ++m) +// { +// IdentityLayer<> identityLayer; +// identityLayer.Forward(input, identityLayer.OutputParameter()); +// +// module3->Add >(identityLayer); +// } +// module3->Forward(input, output3); +// +// MultiplyMerge<> module4(std::move(*module3)); +// delete module3; +// +// module4.Forward(input, output4); +// CheckMatrices(output3, output4); +// } + +// /** +// * Simple Atrous Convolution layer test. +// */ +// TEST_CASE("SimpleAtrousConvolutionLayerTest", "[ANNLayerTest]") +// { +// arma::mat output, input, delta; + +// AtrousConvolution<> module1(1, 1, 3, 3, 1, 1, 0, 0, 7, 7, 2, 2); +// // Test the Forward function. +// input = arma::linspace(0, 48, 49); +// module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); +// module1.Parameters()(0) = 1.0; +// module1.Parameters()(8) = 2.0; +// module1.Reset(); +// module1.Forward(input, output); +// // Value calculated using tensorflow.nn.atrous_conv2d() +// REQUIRE(arma::accu(output) == 792.0); + +// // Test the Backward function. +// module1.Backward(input, output, delta); +// REQUIRE(arma::accu(delta) == 2376); + +// AtrousConvolution<> module2(1, 1, 3, 3, 2, 2, 0, 0, 7, 7, 2, 2); +// // Test the forward function. +// input = arma::linspace(0, 48, 49); +// module2.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); +// module2.Parameters()(0) = 1.0; +// module2.Parameters()(3) = 1.0; +// module2.Parameters()(6) = 1.0; +// module2.Reset(); +// module2.Forward(input, output); +// // Value calculated using tensorflow.nn.conv2d() +// REQUIRE(arma::accu(output) == 264.0); + +// // Test the backward function. +// module2.Backward(input, output, delta); +// REQUIRE(arma::accu(delta) == 792.0); +// } + +// /** +// * Atrous Convolution layer numerical gradient test. +// */ +// TEST_CASE("GradientAtrousConvolutionLayerTest", "[ANNLayerTest]") +// { +// // Add function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::linspace(0, 35, 36)), +// target(arma::mat("0")) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(1, 1, 3, 3, 1, 1, 0, 0, 6, 6, 2, 2); +// model->Add >(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// FFN* model; +// arma::mat input, target; +// } function; + +// // TODO: this tolerance seems far higher than necessary. The implementation +// // should be checked. +// REQUIRE(CheckGradient(function) <= 0.2); +// } + +// /** +// * Test the functions to access and modify the parameters of the +// * AtrousConvolution layer. +// */ +// TEST_CASE("AtrousConvolutionLayerParametersTest", "[ANNLayerTest]") +// { +// // Parameter order for the constructor: inSize, outSize, kW, kH, dW, dH, padW, +// // padH, inputWidth, inputHeight, dilationW, dilationH, paddingType ("none"). +// AtrousConvolution<> layer1(1, 2, 3, 4, 5, 6, std::make_tuple(7, 8), +// std::make_tuple(9, 10), 11, 12, 13, 14); +// AtrousConvolution<> layer2(2, 3, 4, 5, 6, 7, std::make_tuple(8, 9), +// std::make_tuple(10, 11), 12, 13, 14, 15); + +// // Make sure we can get the parameters successfully. +// REQUIRE(layer1.InputWidth() == 11); +// REQUIRE(layer1.InputHeight() == 12); +// REQUIRE(layer1.KernelWidth() == 3); +// REQUIRE(layer1.KernelHeight() == 4); +// REQUIRE(layer1.StrideWidth() == 5); +// REQUIRE(layer1.StrideHeight() == 6); +// REQUIRE(layer1.Padding().PadHTop() == 9); +// REQUIRE(layer1.Padding().PadHBottom() == 10); +// REQUIRE(layer1.Padding().PadWLeft() == 7); +// REQUIRE(layer1.Padding().PadWRight() == 8); +// REQUIRE(layer1.DilationWidth() == 13); +// REQUIRE(layer1.DilationHeight() == 14); + +// // Now modify the parameters to match the second layer. +// layer1.InputWidth() = 12; +// layer1.InputHeight() = 13; +// layer1.KernelWidth() = 4; +// layer1.KernelHeight() = 5; +// layer1.StrideWidth() = 6; +// layer1.StrideHeight() = 7; +// layer1.Padding().PadHTop() = 10; +// layer1.Padding().PadHBottom() = 11; +// layer1.Padding().PadWLeft() = 8; +// layer1.Padding().PadWRight() = 9; +// layer1.DilationWidth() = 14; +// layer1.DilationHeight() = 15; + +// // Now ensure all results are the same. +// REQUIRE(layer1.InputWidth() == layer2.InputWidth()); +// REQUIRE(layer1.InputHeight() == layer2.InputHeight()); +// REQUIRE(layer1.KernelWidth() == layer2.KernelWidth()); +// REQUIRE(layer1.KernelHeight() == layer2.KernelHeight()); +// REQUIRE(layer1.StrideWidth() == layer2.StrideWidth()); +// REQUIRE(layer1.StrideHeight() == layer2.StrideHeight()); +// REQUIRE(layer1.Padding().PadHTop() == layer2.Padding().PadHTop()); +// REQUIRE(layer1.Padding().PadHBottom() == +// layer2.Padding().PadHBottom()); +// REQUIRE(layer1.Padding().PadWLeft() == +// layer2.Padding().PadWLeft()); +// REQUIRE(layer1.Padding().PadWRight() == +// layer2.Padding().PadWRight()); +// REQUIRE(layer1.DilationWidth() == layer2.DilationWidth()); +// REQUIRE(layer1.DilationHeight() == layer2.DilationHeight()); +// } + +// /** +// * Test that the padding options are working correctly in Atrous Convolution +// * layer. +// */ +// TEST_CASE("AtrousConvolutionLayerPaddingTest", "[ANNLayerTest]") +// { +// arma::mat output, input, delta; + +// // Check valid padding option. +// AtrousConvolution<> module1(1, 1, 3, 3, 1, 1, +// std::tuple(1, 1), std::tuple(1, 1), 7, 7, +// 2, 2, "valid"); + +// // Test the Forward function. +// input = arma::linspace(0, 48, 49); +// module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); +// module1.Reset(); +// module1.Forward(input, output); + +// REQUIRE(arma::accu(output) == 0); +// REQUIRE(output.n_rows == 9); +// REQUIRE(output.n_cols == 1); + +// // Test the Backward function. +// module1.Backward(input, output, delta); + +// // Check same padding option. +// AtrousConvolution<> module2(1, 1, 3, 3, 1, 1, +// std::tuple(0, 0), std::tuple(0, 0), 7, 7, +// 2, 2, "same"); + +// // Test the forward function. +// input = arma::linspace(0, 48, 49); +// module2.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); +// module2.Reset(); +// module2.Forward(input, output); + +// REQUIRE(arma::accu(output) == 0); +// REQUIRE(output.n_rows == 49); +// REQUIRE(output.n_cols == 1); + +// // Test the backward function. +// module2.Backward(input, output, delta); +// } + +/** + * Tests the GroupNorm layer. + */ +// TEST_CASE("GroupNormTest", "[ANNLayerTest]") +// { +// arma::mat input, output, backwardOutput; +// input = { +// { 2, 0, 1 }, +// { 3, 1, 2 }, +// { 5, 1, 3 }, +// { 7, 2, 4 }, +// { 11, 3, 5 }, +// { 13, 5, 6 }, +// { 17, 8, 7 }, +// { 19, 13, 8 } +// }; +// +// GroupNorm<> model(2, 4); +// model.Reset(); +// +// model.Forward(input, output); +// arma::mat result; +// result = { +// { -1.1717001972, -1.4142135482, -1.3416407811 }, +// { -0.6509445540, 0.0000000000 , -0.4472135937 }, +// { 0.3905667324 , 0.0000000000 , 0.4472135937 }, +// { 1.4320780188 , 1.4142135482 , 1.341640781 }, +// { -1.2649110634, -1.1283296293, -1.3416407811 }, +// { -0.6324555317, -0.5973509802, -0.4472135937 }, +// { 0.6324555317 , 0.1991169934 , 0.4472135937 }, +// { 1.2649110634 , 1.5265636161 , 1.3416407811 } +// }; +// +// CheckMatrices(output, result, 1e-5); +// } + +/** + * GroupNorm layer numerical gradient test. + */ +// TEST_CASE("GradientGroupNormTest", "[ANNLayerTest]") +// { +// // Add function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randn(10, 256)), +// target(arma::zeros(1, 256)) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(10, 10); +// model->Add >(1, 10); +// model->Add >(10, 2); +// model->Add >(); +// } +// +// ~GradientFunction() +// { +// delete model; +// } +// +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 256, false); +// model->Gradient(model->Parameters(), 0, gradient, 256); +// return error; +// } +// +// arma::mat& Parameters() { return model->Parameters(); } +// +// FFN* model; +// arma::mat input, target; +// } function; +// +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +/** + * Tests the LayerNorm layer. + * +TEST_CASE("LayerNormTest", "[ANNLayerTest]") +{ + arma::mat input, output; + input = { { 5.1, 3.5 }, + { 4.9, 3.0 }, + { 4.7, 3.2 } }; + + LayerNorm model(input.n_rows); + model.Reset(); + + model.Forward(input, output); + arma::mat result; + result = { { 1.2247, 1.2978 }, + { 0, -1.1355 }, + { -1.2247, -0.1622 } }; + + CheckMatrices(output, result, 1e-1); + result.clear(); + + output = model.Mean(); + result = { 4.9000, 3.2333 }; + + CheckMatrices(output, result, 1e-1); + result.clear(); + + output = model.Variance(); + result = { 0.0267, 0.0422 }; + + CheckMatrices(output, result, 1e-1); +} +*/ + +/** + * LayerNorm layer numerical gradient test. + * +TEST_CASE("GradientLayerNormTest", "[ANNLayerTest]") +{ + // Add function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randn(10, 256)), + target(arma::zeros(1, 256)) + { + model = new FFN(); + model->ResetData(input, target); + model->Add(); + model->Add(10, 10); + model->Add(10); + model->Add(10, 2); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 16, false); + model->Gradient(model->Parameters(), 0, gradient, 16); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + REQUIRE(CheckGradient(function) <= 1e-4); +} +*/ + +/** + * Test that the functions that can access the parameters of the + * Layer Norm layer work. + * +TEST_CASE("LayerNormLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : size, eps. + LayerNorm layer(5, 1e-3); + + // Make sure we can get the parameters successfully. + REQUIRE(layer.InSize() == 5); + REQUIRE(layer.Epsilon() == 1e-3); +} +*/ + +// /** +// * Test if the AddMerge layer is able to forward the +// * Forward/Backward/Gradient calls. +// */ +// TEST_CASE("AddMergeRunTest", "[ANNLayerTest]") +// { +// arma::mat output, input, delta, error; + +// AddMerge<> module(true, true); + +// Linear<>* linear = new Linear<>(10, 10); +// module.Add(linear); + +// linear->Parameters().randu(); +// linear->Reset(); + +// input = arma::zeros(10, 1); +// module.Forward(input, output); + +// double parameterSum = arma::accu(linear->Parameters().submat( +// 100, 0, linear->Parameters().n_elem - 1, 0)); + +// // Test the Backward function. +// module.Backward(input, input, delta); + +// // Clean up before we break, +// delete linear; + +// REQUIRE(parameterSum == Approx(arma::accu(output)).epsilon(1e-5)); +// REQUIRE(arma::accu(delta) == 0); +// } + +/** + * Test if the MultiplyMerge layer is able to forward the + * Forward/Backward/Gradient calls. + * +TEST_CASE("MultiplyMergeRunTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta, error; + + MultiplyMerge module(true, true); + + Linear* linear = new Linear(10, 10); + module.Add(linear); + + linear->Parameters().randu(); + linear->Reset(); + + input = arma::zeros(10, 1); + module.Forward(input, output); + + double parameterSum = arma::accu(linear->Parameters().submat( + 100, 0, linear->Parameters().n_elem - 1, 0)); + + // Test the Backward function. + module.Backward(input, input, delta); + + // Clean up before we break, + delete linear; + + REQUIRE(parameterSum == Approx(arma::accu(output)).epsilon(1e-5)); + REQUIRE(arma::accu(delta) == 0); +} +*/ + +/** + * Simple subview module test. + * +TEST_CASE("SimpleSubviewLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta, outputMat; + Subview moduleRow(1, 10, 19); + + // Test the Forward function for a vector. + input = arma::ones(20, 1); + moduleRow.Forward(input, output); + REQUIRE(output.n_rows == 10); + + Subview moduleMat(4, 3, 6, 0, 2); + + // Test the Forward function for a matrix. + input = arma::ones(20, 8); + moduleMat.Forward(input, outputMat); + REQUIRE(outputMat.n_rows == 12); + REQUIRE(outputMat.n_cols == 2); + + // Test the Backward function. + moduleMat.Backward(input, input, delta); + REQUIRE(accu(delta) == 160); + REQUIRE(delta.n_rows == 20); +} +*/ + +/** + * Subview index test. + * +TEST_CASE("SubviewIndexTest", "[ANNLayerTest]") +{ + arma::mat outputEnd, outputMid, outputStart, input, delta; + input = arma::linspace(1, 20, 20); + + // Slicing from the initial indices. + Subview moduleStart(1, 0, 9); + arma::mat subStart = arma::linspace(1, 10, 10); + + moduleStart.Forward(input, outputStart); + CheckMatrices(outputStart, subStart); + + // Slicing from the mid indices. + Subview moduleMid(1, 6, 15); + arma::mat subMid = arma::linspace(7, 16, 10); + + moduleMid.Forward(input, outputMid); + CheckMatrices(outputMid, subMid); + + // Slicing from the end indices. + Subview moduleEnd(1, 10, 19); + arma::mat subEnd = arma::linspace(11, 20, 10); + + moduleEnd.Forward(input, outputEnd); + CheckMatrices(outputEnd, subEnd); +} +*/ + +/** + * Subview batch test. + * +TEST_CASE("SubviewBatchTest", "[ANNLayerTest]") +{ + arma::mat output, input, outputCol, outputMat, outputDef; + + // All rows selected. + Subview moduleCol(1, 0, 19); + + // Test with inSize 1. + input = arma::ones(20, 8); + moduleCol.Forward(input, outputCol); + CheckMatrices(outputCol, input); + + // Few rows and columns selected. + Subview moduleMat(4, 3, 6, 0, 2); + + // Test with inSize greater than 1. + moduleMat.Forward(input, outputMat); + output = arma::ones(12, 2); + CheckMatrices(outputMat, output); + + // endCol changed to 3 by default. + Subview moduleDef(4, 1, 6, 0, 4); + + // Test with inSize greater than 1 and endCol >= inSize. + moduleDef.Forward(input, outputDef); + output = arma::ones(24, 2); + CheckMatrices(outputDef, output); +} +*/ + +/** + * Test that the functions that can modify and access the parameters of the + * Subview layer work. + * +TEST_CASE("SubviewLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : inSize, beginRow, endRow, beginCol, endCol. + Subview layer1(1, 2, 3, 4, 5); + Subview layer2(1, 3, 4, 5, 6); + + // Make sure we can get the parameters correctly. + REQUIRE(layer1.InSize() == 1); + REQUIRE(layer1.BeginRow() == 2); + REQUIRE(layer1.EndRow() == 3); + REQUIRE(layer1.BeginCol() == 4); + REQUIRE(layer1.EndCol() == 5); + + // Now modify the parameters to match the second layer. + layer1.BeginRow() = 3; + layer1.EndRow() = 4; + layer1.BeginCol() = 5; + layer1.EndCol() = 6; + + // Now ensure all results are the same. + REQUIRE(layer1.InSize() == layer2.InSize()); + REQUIRE(layer1.BeginRow() == layer2.BeginRow()); + REQUIRE(layer1.EndRow() == layer2.EndRow()); + REQUIRE(layer1.BeginCol() == layer2.BeginCol()); + REQUIRE(layer1.EndCol() == layer2.EndCol()); +} +*/ + +/* + * Simple Reparametrization module test. + * +TEST_CASE("SimpleReparametrizationLayerTest", "[ANNLayerTest]") +{ + arma::mat input, output, delta; + Reparametrization module(5); + + // Test the Forward function. As the mean is zero and the standard + // deviation is small, after multiplying the gaussian sample, the + // output should be small enough. + input = join_cols(arma::ones(5, 1) * -15, + arma::zeros(5, 1)); + module.Forward(input, output); + REQUIRE(arma::accu(output) <= 1e-5); + + // Test the Backward function. + arma::mat gy = arma::zeros(5, 1); + module.Backward(input, gy, delta); + REQUIRE(arma::accu(delta) != 0); // klBackward will be added. +} +*/ + +/** + * Reparametrization module stochastic boolean test. + * +TEST_CASE("ReparametrizationLayerStochasticTest", "[ANNLayerTest]") +{ + arma::mat input, outputA, outputB; + Reparametrization module(5, false); + + input = join_cols(arma::ones(5, 1), + arma::zeros(5, 1)); + + // Test if two forward passes generate same output. + module.Forward(input, outputA); + module.Forward(input, outputB); + + CheckMatrices(outputA, outputB); +} +*/ + +/** + * Reparametrization module includeKl boolean test. + * +TEST_CASE("ReparametrizationLayerIncludeKlTest", "[ANNLayerTest]") +{ + arma::mat input, output, gy, delta; + Reparametrization module(5, true, false); + + input = join_cols(arma::ones(5, 1), + arma::zeros(5, 1)); + module.Forward(input, output); + + // As KL divergence is not included, with the above inputs, the delta + // matrix should be all zeros. + gy = arma::zeros(output.n_rows, output.n_cols); + module.Backward(output, gy, delta); + + REQUIRE(arma::accu(delta) == 0); +} +*/ + +/** + * Jacobian Reparametrization module test. + * +TEST_CASE("JacobianReparametrizationLayerTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t inputElementsHalf = RandInt(2, 10); + + arma::mat input; + input.set_size(inputElementsHalf * 2, 1); + + Reparametrization module(inputElementsHalf, false, false); + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} +*/ + +/** + * Reparametrization layer numerical gradient test. + * +TEST_CASE("GradientReparametrizationLayerTest", "[ANNLayerTest]") +{ + // Linear function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(10, 1)), + target(arma::mat("0")) + { + model = new FFN(); + model->ResetData(input, target); + model->Add(); + model->Add(10, 6); + model->Add(3, false, true, 1); + model->Add(3, 2); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + // REQUIRE(CheckGradient(function) <= 1e-4); +} +*/ + +/** + * Reparametrization layer beta numerical gradient test. + * +TEST_CASE("GradientReparametrizationLayerBetaTest", "[ANNLayerTest]") +{ + // Linear function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(10, 2)), + target(arma::mat("0 0")) + { + model = new FFN(); + model->ResetData(input, target); + model->Add(); + model->Add(10, 6); + // Use a value of beta not equal to 1. + model->Add(3, false, true, 2); + model->Add(3, 2); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + // REQUIRE(CheckGradient(function) <= 1e-4); +} +*/ + +/** + * Test that the functions that can access the parameters of the + * Reparametrization layer work. + * +TEST_CASE("ReparametrizationLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : latentSize, stochastic, includeKL, beta. + Reparametrization layer(5, false, false, 2); + + // Make sure we can get the parameters successfully. + REQUIRE(layer.OutputSize() == 5); + REQUIRE(layer.Stochastic() == false); + REQUIRE(layer.IncludeKL() == false); + REQUIRE(layer.Beta() == 2); +} +*/ + +/** + * Simple residual module test. + * +TEST_CASE("SimpleResidualLayerTest", "[ANNLayerTest]") +{ + arma::mat outputA, outputB, input, deltaA, deltaB; + + Sequential* sequential = new Sequential(true); + Residual* residual = new Residual(true); + + Linear* linearA = new Linear(10, 10); + linearA->Parameters().randu(); + linearA->Reset(); + Linear* linearB = new Linear(10, 10); + linearB->Parameters().randu(); + linearB->Reset(); + + // Add the same layers (with the same parameters) to both Sequential and + // Residual object. + sequential->Add(linearA); + sequential->Add(linearB); + + residual->Add(linearA); + residual->Add(linearB); + + // Test the Forward function (pass the same input to both). + input = arma::randu(10, 1); + sequential->Forward(input, outputA); + residual->Forward(input, outputB); + + CheckMatrices(outputA, outputB - input); + + // Test the Backward function (pass the same error to both). + sequential->Backward(input, input, deltaA); + residual->Backward(input, input, deltaB); + + CheckMatrices(deltaA, deltaB - input); + + delete sequential; + delete residual; + delete linearA; + delete linearB; +} +*/ + +/** + * Simple Highway module test. + * +TEST_CASE("SimpleHighwayLayerTest", "[ANNLayerTest]") +{ + arma::mat outputA, outputB, input, deltaA, deltaB; + Sequential* sequential = new Sequential(true); + Highway* highway = new Highway(10, true); + highway->Parameters().zeros(); + highway->Reset(); + + Linear* linearA = new Linear(10, 10); + linearA->Parameters().randu(); + linearA->Reset(); + Linear* linearB = new Linear(10, 10); + linearB->Parameters().randu(); + linearB->Reset(); + + // Add the same layers (with the same parameters) to both Sequential and + // Highway object. + highway->Add(linearA); + highway->Add(linearB); + sequential->Add(linearA); + sequential->Add(linearB); + + // Test the Forward function (pass the same input to both). + input = arma::randu(10, 1); + sequential->Forward(input, outputA); + highway->Forward(input, outputB); + + CheckMatrices(outputB, input * 0.5 + outputA * 0.5); + + delete sequential; + delete highway; + delete linearA; + delete linearB; +} +*/ + +/** + * Test that the function that can access the inSize parameter of the + * Highway layer works. + * +TEST_CASE("HighwayLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : inSize, model. + Highway layer(1, true); + + // Make sure we can get the parameter successfully. + REQUIRE(layer.InSize() == 1); +} +*/ + +// /** +// * Sequential layer numerical gradient test. +// */ +// TEST_CASE("GradientHighwayLayerTest", "[ANNLayerTest]") +// { +// // Linear function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(5, 1)), +// target(arma::mat("0")) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add(); +// model->Add(5, 10); + +// highway = new Highway(10); +// highway->Add(10, 10); +// highway->Add(); +// highway->Add(10, 10); +// highway->Add(); + +// model->Add(highway); +// model->Add(10, 2); +// model->Add(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// FFN* model; +// Highway* highway; +// arma::mat input, target; +// } function; + +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +/** + * Sequential layer numerical gradient test. + */ +// TEST_CASE("GradientSequentialLayerTest", "[ANNLayerTest]") +// { +// // Linear function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(10, 1)), +// target(arma::mat("0")) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add(); +// model->Add(10, 10); +// sequential = new Sequential(); +// sequential->Add(10, 10); +// sequential->Add(); +// sequential->Add(10, 5); +// sequential->Add(); + +// model->Add(sequential); +// model->Add(5, 2); +// model->Add(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// FFN* model; +// Sequential* sequential; +// arma::mat input, target; +// } function; + +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +// /** +// * WeightNorm layer numerical gradient test. +// */ +// TEST_CASE("GradientWeightNormLayerTest", "[ANNLayerTest]") +// { +// // Linear function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(10, 1)), +// target(arma::mat("0")) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add(10, 10); + +// Linear* linear = new Linear(10, 2); +// weightNorm = new WeightNorm(linear); + +// model->Add(weightNorm); +// model->Add(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// FFN* model; +// WeightNorm* weightNorm; +// arma::mat input, target; +// } function; + +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +// /** +// * Test if the WeightNorm layer is able to forward the +// * Forward/Backward/Gradient calls. +// */ +// TEST_CASE("WeightNormRunTest", "[ANNLayerTest]") +// { +// arma::mat output, input, delta, error; +// Linear* linear = new Linear(10, 10); + +// WeightNorm module(linear); + +// module.Parameters().randu(); +// module.Reset(); + +// linear->Bias().zeros(); + +// input = arma::zeros(10, 1); +// module.Forward(input, output); + +// // Test the Backward function. +// module.Backward(input, input, delta); + +// REQUIRE(0 == arma::accu(output)); +// REQUIRE(arma::accu(delta) == 0); +// } + +// /** +// * Simple serialization test for layer normalization layer. +// */ +// TEST_CASE("LayerNormSerializationTest", "[ANNLayerTest]") +// { +// LayerNorm<> layer(10); +// ANNLayerSerializationTest(layer); +// } + +/** + * Test that the padding options in Transposed Convolution layer. + * +TEST_CASE("TransposedConvolutionLayerPaddingTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + + TransposedConvolution module1(1, 1, 3, 3, 1, 1, 0, 0, 4, 4, 6, 6, "VALID"); + // Test the forward function. + // Valid Should give the same result. + input = arma::linspace(0, 15, 16); + module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module1.Reset(); + module1.Forward(input, output); + // Value calculated using tensorflow.nn.conv2d_transpose(). + REQUIRE(arma::accu(output) == 0.0); + + // Test the Backward Function. + module1.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 0.0); + + // Test Valid for non zero padding. + TransposedConvolution module2(1, 1, 3, 3, 2, 2, + std::tuple(0, 0), std::tuple(0, 0), + 2, 2, 5, 5, "VALID"); + // Test the forward function. + input = arma::linspace(0, 3, 4); + module2.Parameters() = arma::mat(25 + 1, 1, arma::fill::zeros); + module2.Parameters()(2) = 8.0; + module2.Parameters()(4) = 6.0; + module2.Parameters()(6) = 4.0; + module2.Parameters()(8) = 2.0; + module2.Reset(); + module2.Forward(input, output); + // Value calculated using torch.nn.functional.conv_transpose2d(). + REQUIRE(arma::accu(output) == 120.0); + + // Test the Backward Function. + module2.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 960.0); + + // Test for same padding type. + TransposedConvolution module3(1, 1, 3, 3, 2, 2, 0, 0, 3, 3, 3, 3, "SAME"); + // Test the forward function. + input = arma::linspace(0, 8, 9); + module3.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module3.Reset(); + module3.Forward(input, output); + REQUIRE(arma::accu(output) == 0); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + + // Test the Backward Function. + module3.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 0.0); + + // Output shape should equal input. + TransposedConvolution module4(1, 1, 3, 3, 1, 1, + std::tuple(2, 2), std::tuple(2, 2), + 5, 5, 5, 5, "SAME"); + // Test the forward function. + input = arma::linspace(0, 24, 25); + module4.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module4.Reset(); + module4.Forward(input, output); + REQUIRE(arma::accu(output) == 0); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + + // Test the Backward Function. + module4.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 0.0); + + TransposedConvolution module5(1, 1, 3, 3, 2, 2, 0, 0, 2, 2, 2, 2, "SAME"); + // Test the forward function. + input = arma::linspace(0, 3, 4); + module5.Parameters() = arma::mat(25 + 1, 1, arma::fill::zeros); + module5.Reset(); + module5.Forward(input, output); + REQUIRE(arma::accu(output) == 0); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + + // Test the Backward Function. + module5.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 0.0); + + TransposedConvolution module6(1, 1, 4, 4, 1, 1, 1, 1, 5, 5, 5, 5, "SAME"); + // Test the forward function. + input = arma::linspace(0, 24, 25); + module6.Parameters() = arma::mat(16 + 1, 1, arma::fill::zeros); + module6.Reset(); + module6.Forward(input, output); + REQUIRE(arma::accu(output) == 0); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + + // Test the Backward Function. + module6.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 0.0); +} +*/ + +/** + * Simple test for Lp Pooling layer. + */ +// TEST_CASE("LpMaxPoolingTestCase", "[ANNLayerTest]") +// { +// // For rectangular input to pooling layers. +// arma::mat input = arma::mat(8, 1); +// arma::mat output; +// input.zeros(); +// input(0) = input(6) = 30; +// input(1) = input(7) = 120; +// input(2) = input(4) = 272; +// input(3) = input(5) = 315; +// // Output-Size should be 1 x 2. +// // Square output. +// LpPooling<> module1(4, 2, 2, 2, 2); +// module1.InputHeight() = 2; +// module1.InputWidth() = 4; +// module1.Forward(input, output); +// // Calculated using torch.nn.LPPool2d(). +// REQUIRE(arma::accu(output) - 706.0 == Approx(0.0).margin(2e-5)); +// REQUIRE(output.n_elem == 2); +// +// // For Square input. +// input = arma::mat(16, 1); +// input.zeros(); +// input(0) = 4; +// input(1) = 3; +// input(3) = 12; +// input(7) = 35; +// input(8) = 6; +// input(11) = 7; +// input(12) = 8; +// input(15) = 24; +// // Output-Size should be 2 x 2. +// // Square output. +// LpPooling<> module3(2, 2, 2, 2, 2); +// module3.InputHeight() = 4; +// module3.InputWidth() = 4; +// module3.Forward(input, output); +// // Calculated using torch.nn.LPPool2d(). +// REQUIRE(arma::accu(output) - 77.0 == Approx(0.0).margin(2e-5)); +// REQUIRE(output.n_elem == 4); +// } + +/** + * Test that the functions that can modify and access the parameters of the + * Glimpse layer work. + * +TEST_CASE("GlimpseLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : inSize, size, depth, scale, inputWidth, inputHeight. + Glimpse layer1(1, 2, 3, 4, 5, 6); + Glimpse layer2(1, 2, 3, 4, 6, 7); + + // Make sure we can get the parameters successfully. + REQUIRE(layer1.InputHeight() == 6); + REQUIRE(layer1.InputWidth() == 5); + REQUIRE(layer1.Scale() == 4); + REQUIRE(layer1.Depth() == 3); + REQUIRE(layer1.GlimpseSize() == 2); + REQUIRE(layer1.InSize() == 1); + + // Now modify the parameters to match the second layer. + layer1.InputHeight() = 7; + layer1.InputWidth() = 6; + + // Now ensure that all the results are the same. + REQUIRE(layer1.InputHeight() == layer2.InputHeight()); + REQUIRE(layer1.InputWidth() == layer2.InputWidth()); + REQUIRE(layer1.Scale() == layer2.Scale()); + REQUIRE(layer1.Depth() == layer2.Depth()); + REQUIRE(layer1.GlimpseSize() == layer2.GlimpseSize()); + REQUIRE(layer1.InSize() == layer2.InSize()); +} +*/ + +/** + * Test that the function that can access the stdev parameter of the + * Reinforce Normal layer works. + * +TEST_CASE("ReinforceNormalLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter : stdev. + ReinforceNormal layer(4.0); + + // Make sure we can get the parameter successfully. + REQUIRE(layer.StandardDeviation() == 4.0); +} +*/ + +/* +TEST_CASE("TransposedConvolutionalLayerOptionalParameterTest", "[ANNLayerTest]") +{ + Sequential* decoder = new Sequential(); + + // Check if we can create an object without specifying output. + REQUIRE_NOTHROW(decoder->Add(24, 16, + 5, 5, 1, 1, 0, 0, 10, 10)); + + REQUIRE_NOTHROW(decoder->Add(16, 1, + 15, 15, 1, 1, 1, 1, 14, 14)); + + delete decoder; +} +*/ + +// /** +// * Linear module weight initialization test. +// */ +// TEST_CASE("LinearLayerWeightInitializationTest", "[ANNLayerTest]") +// { +// size_t inSize = 10, outSize = 4; +// Linear<> linear = Linear<>(inSize, outSize); +// linear.Reset(); +// RandomInitialization().Initialize(linear.Weight()); +// linear.Bias().ones(); + +// REQUIRE(std::equal(linear.Weight().begin(), +// linear.Weight().end(), linear.Parameters().begin())); + +// REQUIRE(std::equal(linear.Bias().begin(), +// linear.Bias().end(), linear.Parameters().begin() + inSize * outSize)); + +// REQUIRE(linear.Weight().n_rows == outSize); +// REQUIRE(linear.Weight().n_cols == inSize); +// REQUIRE(linear.Bias().n_rows == outSize); +// REQUIRE(linear.Bias().n_cols == 1); +// REQUIRE(linear.Parameters().n_rows == inSize * outSize + outSize); +// } + +// /** +// * Atrous Convolution module weight initialization test. +// */ +// TEST_CASE("AtrousConvolutionLayerWeightInitializationTest", "[ANNLayerTest]") +// { +// size_t inSize = 2, outSize = 3; +// size_t kernelWidth = 4, kernelHeight = 5; +// AtrousConvolution<> module = AtrousConvolution<>(inSize, outSize, +// kernelWidth, kernelHeight, 6, 7, std::make_tuple(8, 9), +// std::make_tuple(10, 11), 12, 13, 14, 15); +// module.Reset(); +// RandomInitialization().Initialize(module.Weight()); +// module.Bias().ones(); + +// REQUIRE(std::equal(module.Weight().begin(), +// module.Weight().end(), module.Parameters().begin())); + +// REQUIRE(std::equal(module.Bias().begin(), +// module.Bias().end(), module.Parameters().end() - outSize)); + +// REQUIRE(module.Weight().n_rows == kernelWidth); +// REQUIRE(module.Weight().n_cols == kernelHeight); +// REQUIRE(module.Weight().n_slices == inSize * outSize); +// REQUIRE(module.Bias().n_rows == outSize); +// REQUIRE(module.Bias().n_cols == 1); +// REQUIRE(module.Parameters().n_rows +// == (outSize * inSize * kernelWidth * kernelHeight) + outSize); +// } + +/** + * Transposed Convolution module weight initialization test. + * +TEST_CASE("TransposedConvolutionWeightInitializationTest", "[ANNLayerTest]") +{ + size_t inSize = 3, outSize = 3; + size_t kernelWidth = 4, kernelHeight = 4; + TransposedConvolution module = TransposedConvolution(inSize, outSize, + kernelWidth, kernelHeight, 1, 1, 1, 1, 5, 5, 6, 6); + module.Reset(); + RandomInitialization().Initialize(module.Weight()); + module.Bias().ones(); + + REQUIRE(std::equal(module.Weight().begin(), + module.Weight().end(), module.Parameters().begin())); + + REQUIRE(std::equal(module.Bias().begin(), + module.Bias().end(), module.Parameters().end() - outSize)); + + REQUIRE(module.Weight().n_rows == kernelWidth); + REQUIRE(module.Weight().n_cols == kernelHeight); + REQUIRE(module.Weight().n_slices == inSize * outSize); + REQUIRE(module.Bias().n_rows == outSize); + REQUIRE(module.Bias().n_cols == 1); + REQUIRE(module.Parameters().n_rows + == (outSize * inSize * kernelWidth * kernelHeight) + outSize); +} +*/ + +/** + * Simple Test for ChannelShuffle layer. + */ +// TEST_CASE("ChannelShuffleLayerTest", "[ANNLayerTest]") +// { +// arma::mat input1, output1, outputExpected1, outputBackward1; +// ChannelShuffle<> module1(2, 2, 6, 2); +// +// input1 << 1 << 13 << arma::endr +// << 2 << 14 << arma::endr +// << 3 << 15 << arma::endr +// << 4 << 16 << arma::endr +// << 5 << 17 << arma::endr +// << 6 << 18 << arma::endr +// << 7 << 19 << arma::endr +// << 8 << 20 << arma::endr +// << 9 << 21 << arma::endr +// << 10 << 22 << arma::endr +// << 11 << 23 << arma::endr +// << 12 << 24 << arma::endr; +// input1.reshape(24, 1); +// // Value calculated using torch.nn.ChannelShuffle(). +// outputExpected1 << 1 << 17 << arma::endr +// << 2 << 18 << arma::endr +// << 3 << 19 << arma::endr +// << 4 << 20 << arma::endr +// << 13 << 9 << arma::endr +// << 14 << 10 << arma::endr +// << 15 << 11 << arma::endr +// << 16 << 12 << arma::endr +// << 5 << 21 << arma::endr +// << 6 << 22 << arma::endr +// << 7 << 23 << arma::endr +// << 8 << 24 << arma::endr; +// outputExpected1.reshape(24, 1); +// // Check the Forward pass of the layer. +// module1.Forward(input1, output1); +// CheckMatrices(output1, outputExpected1); +// +// // Check the Backward pass of the layer. +// module1.Backward(output1, output1, outputBackward1); +// CheckMatrices(input1, outputBackward1); +// +// } + +/** + * Simple Test for PixelShuffle layer. + */ +// TEST_CASE("PixelShuffleLayerTest", "[ANNLayerTest]") +// { +// arma::mat input1, output1, gy1, g1, outputExpected1, gExpected1; +// arma::mat input2, output2, gy2, g2, outputExpected2, gExpected2; +// PixelShuffle<> module1(2, 2, 2, 4); +// PixelShuffle<> module2(2, 2, 2, 4); +// +// // Input is a single image, of size (2,2) and having 4 channels. +// input1 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 +// << 0 << 0 << arma::endr; +// gy1 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8 +// << 12 << 16 << arma::endr; +// +// // Calculated using torch.nn.PixelShuffle(). +// outputExpected1 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0 +// << 0 << 0 << 0 << 0 << arma::endr; +// gExpected1 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12 +// << 6 << 14 << 8 << 16 << arma::endr; +// +// input1 = input1.t(); +// outputExpected1 = outputExpected1.t(); +// gy1 = gy1.t(); +// gExpected1 = gExpected1.t(); +// +// // Check the Forward pass of the layer. +// module1.Forward(input1, output1); +// CheckMatrices(output1, outputExpected1); +// +// // Check the Backward pass of the layer. +// module1.Backward(input1, gy1, g1); +// CheckMatrices(g1, gExpected1); +// +// // Input is a batch of 2 images, each of size (2,2) and having 4 channels. +// input2 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 +// << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0 +// << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr; +// gy2 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8 +// << 12 << 16 << arma::endr << 17 << 21 << 25 << 29 << 18 << 22 << 26 << 30 +// << 19 << 23 << 27 << 31 << 20 << 24 << 28 << 32 << arma::endr; +// +// // Calculated using torch.nn.PixelShuffle(). +// outputExpected2 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0 +// << 0 << 0 << 0 << 0 << arma::endr << 5 << 0 << 7 << 0 << 0 << 0 << 0 << 0 +// << 6 << 0 << 8 << 0 << 0 << 0 << 0 << 0 << arma::endr; +// gExpected2 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12 +// << 6 << 14 << 8 << 16 << arma::endr << 17 << 25 << 19 << 27 << 21 << 29 +// << 23 << 31 << 18 << 26 << 20 << 28 << 22 << 30 << 24 << 32 << arma::endr; +// +// input2 = input2.t(); +// outputExpected2 = outputExpected2.t(); +// gy2 = gy2.t(); +// gExpected2 = gExpected2.t(); +// +// // Check the Forward pass of the layer. +// module2.Forward(input2, output2); +// CheckMatrices(output2, outputExpected2); +// +// // Check the Backward pass of the layer. +// module2.Backward(input2, gy2, g2); +// CheckMatrices(g2, gExpected2); +// } + +/** + * Test that the function that can access the parameters of the + * PixelShuffle layer works. + */ +// TEST_CASE("PixelShuffleLayerParametersTest", "[ANNLayerTest]") +// { +// // Create the layer using the empty constructor. +// PixelShuffle<> layer; +// +// // Set the different input parameters of the layer. +// layer.UpscaleFactor() = 2; +// layer.InputHeight() = 2; +// layer.InputWidth() = 2; +// layer.InputChannels() = 4; +// +// // Make sure we can get the parameters successfully. +// REQUIRE(layer.UpscaleFactor() == 2); +// REQUIRE(layer.InputHeight() == 2); +// REQUIRE(layer.InputWidth() == 2); +// REQUIRE(layer.InputChannels() == 4); +// +// arma::mat input, output; +// // Input is a batch of 2 images, each of size (2,2) and having 4 channels. +// input << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 +// << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0 +// << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr; +// input = input.t(); +// layer.Forward(input, output); +// +// // Check whether output parameters are returned correctly. +// REQUIRE(layer.OutputHeight() == 4); +// REQUIRE(layer.OutputWidth() == 4); +// REQUIRE(layer.OutputChannels() == 1); +// } + +// /** +// * Simple Test for SpatialDropout layer. +// */ +// TEST_CASE("SpatialDropoutLayerTest", "[ANNLayerTest]") +// { +// arma::mat input, output, gy, g, temp; +// arma::mat outputsExpected = arma::zeros(8, 12); +// arma::mat gsExpected = arma::zeros(8, 12); + +// // Set the seed to a random value. +// arma::arma_rng::set_seed_random(); +// SpatialDropout<> module(3, 0.2); + +// // Input is a batch of 2 images, each of size (2,2) and having 4 channels. +// input = { 0.4963, 0.0885, 0.7682, 0.1320, 0.3074, 0.4901, 0.6341, 0.8964, +// 0.4556, 0.3489, 0.6323, 0.4017 }; +// +// gy = { 1, 3, 2, 4, 5, 7, 6, 8, 9, 11, 10, 12 }; +// +// // Following values have been calculated using torch.nn.Dropout2d(p=0.2). +// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; +// outputsExpected.row(0) = temp; +// temp = { 0, 0, 0, 0, 0.3842, 0.6126, 0.7926, 1.1205, 0.5695, 0.4361, 0.7904, +// 0.5021 }; +// outputsExpected.row(1) = temp; +// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0, 0, 0, 0, 0.5695, 0.4361, +// 0.7904, 0.5021 }; +// outputsExpected.row(2) = temp; +// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0.3842, 0.6126, 0.7926, 1.1205, 0, +// 0, 0, 0 }; +// outputsExpected.row(3) = temp; +// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 0.5695, 0.4361, 0.7904, 0.5021 }; +// outputsExpected.row(4) = temp; +// temp = { 0, 0, 0, 0, 0.3842, 0.6126, 0.7926, 1.1205, 0, 0, 0, 0 }; +// outputsExpected.row(5) = temp; +// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0, 0, 0, 0, 0, 0, 0, 0 }; +// outputsExpected.row(6) = temp; +// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0.3842, 0.6126, 0.7926, 1.1205, +// 0.5695, 0.4361, 0.7904, 0.5021 }; +// outputsExpected.row(7) = temp; +// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; +// gsExpected.row(0) = temp; +// temp = { 0, 0, 0, 0, 6.2500, 8.7500, 7.5000, 10.0000, 11.2500, 13.7500, +// 12.5000, 15.0000 }; +// gsExpected.row(1) = temp; +// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 0, 0, 0, 0, 11.2500, 13.7500, +// 12.5000, 15.0000 }; +// gsExpected.row(2) = temp; +// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 6.2500, 8.7500, 7.5000, 10.0000, 0, +// 0, 0, 0 }; +// gsExpected.row(3) = temp; +// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 11.2500, 13.7500, 12.5000, 15.0000 }; +// gsExpected.row(4) = temp; +// temp = { 0, 0, 0, 0, 6.2500, 8.7500, 7.5000, 10.0000, 0, 0, 0, 0 }; +// gsExpected.row(5) = temp; +// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 0, 0, 0, 0, 0, 0, 0, 0 }; +// gsExpected.row(6) = temp; +// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 6.2500, 8.7500, 7.5000, 10.0000, +// 11.2500, 13.7500, 12.5000, 15.0000 }; +// gsExpected.row(7) = temp; + +// input = input.t(); +// gy = gy.t(); +// outputsExpected = outputsExpected.t(); +// gsExpected = gsExpected.t(); + +// // Compute the Forward and Backward passes and store the results. +// module.Forward(input, output); +// module.Backward(input, gy, g); + +// // Check through all possible cases, to find a match and then compare results. +// for (size_t i = 0; i < outputsExpected.n_cols; ++i) +// { +// if (arma::approx_equal(outputsExpected.col(i), output, "absdiff", 1e-1)) +// { +// // Check the correctness of the Forward pass of the layer. +// CheckMatrices(output, outputsExpected.col(i), 1e-1); +// // Check the correctness of the Backward pass of the layer. +// CheckMatrices(g, gsExpected.col(i), 1e-1); +// } +// } + +// // Check if the output is same as input when using deterministic mode. +// module.Deterministic() = true; +// output.clear(); +// module.Forward(input, output); +// CheckMatrices(output, input, 1e-1); +// } + +// /** +// * Test that the function that can access the parameters of the +// * SpatialDropout layer works. +// */ +// TEST_CASE("SpatialDropoutLayerParametersTest", "[ANNLayerTest]") +// { +// // Create the layer using the empty constructor. +// SpatialDropout<> layer; + +// // Set the input parameters. +// layer.Size() = 3; +// layer.Ratio(0.2); + +// // Check whether the input parameters have been set correctly. +// REQUIRE(layer.Size() == 3); +// REQUIRE(layer.Ratio() == 0.2); +// } + +/** + * Simple Positional Encoding layer test. + * +TEST_CASE("SimplePositionalEncodingTest", "[ANNLayerTest]") +{ + const size_t seqLength = 5; + const size_t embedDim = 4; + const size_t batchSize = 2; + + arma::mat input = arma::randu(embedDim * seqLength, batchSize); + arma::mat gy = 0.01 * arma::randu(embedDim * seqLength, batchSize); + arma::mat output, g; + + PositionalEncoding module(embedDim, seqLength); + + // Check Forward function. + module.Forward(input, output); + arma::mat pe = output - input; + CheckMatrices(arma::mean(pe, 1), module.Encoding()); + + // Check Backward function. + module.Backward(input, gy, g); + REQUIRE(std::equal(gy.begin(), gy.end(), g.begin())); +} +*/ + +/** + * Jacobian test for Positional Encoding layer. + * +TEST_CASE("JacobianPositionalEncodingTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t embedDim = 4; + const size_t seqLength = RandInt(5, 10); + arma::mat input; + input.set_size(embedDim * seqLength, 1); + + PositionalEncoding module(embedDim, seqLength); + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} +*/ + +/** + * Simple Multihead Attention test. + * +TEST_CASE("SimpleMultiheadAttentionTest", "[ANNLayerTest]") +{ + size_t tLen = 5; + size_t sLen = tLen; + size_t embedDim = 4; + size_t numHeads = 2; + size_t bsz = 3; + + arma::mat query = 0.1 * arma::randu(embedDim * tLen, bsz); + arma::mat output; + + arma::mat attnMask = arma::zeros(tLen, sLen); + for (size_t i = 0; i < tLen; ++i) + { + for (size_t j = 0; j < sLen; ++j) + { + if (i < j) + attnMask(i, j) = std::numeric_limits::lowest(); + } + } + + arma::mat keyPaddingMask = arma::zeros(1, sLen); + keyPaddingMask(sLen - 1) = std::numeric_limits::lowest(); + + MultiheadAttention module(tLen, sLen, embedDim, numHeads); + module.AttentionMask() = attnMask; + module.KeyPaddingMask() = keyPaddingMask; + module.Reset(); + module.Parameters().randu(); + + // Forward test. + arma::mat input = arma::join_cols(arma::join_cols(query, query), query); + + module.Forward(input, output); + REQUIRE(output.n_rows == embedDim * tLen); + REQUIRE(output.n_cols == bsz); + + // Backward test. + arma::mat gy = 0.01 * arma::randu(embedDim * tLen, bsz); + arma::mat g; + module.Backward(input, gy, g); + REQUIRE(g.n_rows == input.n_rows); + REQUIRE(g.n_cols == input.n_cols); + + // Gradient test. + arma::mat error = 0.05 * arma::randu(embedDim * tLen, bsz); + arma::mat gradient; + module.Gradient(input, error, gradient); + REQUIRE(gradient.n_rows == module.Parameters().n_rows); + REQUIRE(gradient.n_cols == module.Parameters().n_cols); +} +*/ + +/** + * Jacobian MultiheadAttention module test. + * +TEST_CASE("JacobianMultiheadAttentionTest", "[ANNLayerTest]") +{ + // Check when query = key = value. + for (size_t i = 0; i < 5; ++i) + { + const size_t tgtSeqLen = 2; + const size_t embedDim = 4; + const size_t nHeads = 2; + const size_t batchSize = 1; + + arma::mat query = arma::randu(embedDim * tgtSeqLen, batchSize); + arma::mat input = arma::join_cols(arma::join_cols(query, query), query); + + MultiheadAttention module(tgtSeqLen, tgtSeqLen, embedDim, nHeads); + module.Parameters().randu(); + + double error = CustomJacobianTest(module, input); + REQUIRE(error <= 1e-5); + } + + // Check when key = value. + for (size_t i = 0; i < 5; ++i) + { + const size_t tgtSeqLen = 2; + const size_t srcSeqLen = RandInt(2, 5); + const size_t embedDim = 4; + const size_t nHeads = 2; + const size_t batchSize = 1; + + arma::mat query = arma::randu(embedDim * tgtSeqLen, batchSize); + arma::mat key = 0.091 * arma::randu(embedDim * srcSeqLen, batchSize); + arma::mat input = arma::join_cols(arma::join_cols(query, key), key); + + MultiheadAttention module(tgtSeqLen, srcSeqLen, embedDim, nHeads); + module.Parameters().randu(); + + double error = CustomJacobianTest(module, input); + REQUIRE(error <= 1e-5); + } + + // Check when query, key and value are not same. + for (size_t i = 0; i < 5; ++i) + { + const size_t tgtSeqLen = 2; + const size_t srcSeqLen = RandInt(2, 5); + const size_t embedDim = 4; + const size_t nHeads = 2; + const size_t batchSize = 1; + + arma::mat query = arma::randu(embedDim * tgtSeqLen, batchSize); + arma::mat key = 0.091 * arma::randu(embedDim * srcSeqLen, batchSize); + arma::mat value = 0.045 * arma::randu(embedDim * srcSeqLen, batchSize); + arma::mat input = arma::join_cols(arma::join_cols(query, key), value); + + MultiheadAttention module(tgtSeqLen, srcSeqLen, embedDim, nHeads); + module.Parameters().randu(); + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} +*/ + +/** + * Numerical gradient test for MultiheadAttention layer. + * +TEST_CASE("GradientMultiheadAttentionTest", "[ANNLayerTest]") +{ + struct GradientFunction + { + GradientFunction() : + tgtSeqLen(2), + srcSeqLen(2), + embedDim(4), + nHeads(2), + vocabSize(5), + batchSize(2) + { + input = arma::randu(embedDim * (tgtSeqLen + 2 * srcSeqLen), batchSize); + target = arma::zeros(vocabSize, batchSize); + for (size_t i = 0; i < target.n_elem; ++i) + { + const size_t label = RandInt(1, vocabSize); + target(i) = label; + } + + attnMask = arma::zeros(tgtSeqLen, srcSeqLen); + for (size_t i = 0; i < tgtSeqLen; ++i) + { + for (size_t j = 0; j < srcSeqLen; ++j) + { + if (i < j) + attnMask(i, j) = std::numeric_limits::lowest(); + } + } + + keyPaddingMask = arma::zeros(1, srcSeqLen); + keyPaddingMask(srcSeqLen - 1) = std::numeric_limits::lowest(); + + model = new FFN(); + model->ResetData(input, target); + // attnModule = new MultiheadAttention(tgtSeqLen, srcSeqLen, embedDim, + // nHeads); + // attnModule->AttentionMask() = attnMask; + // attnModule->KeyPaddingMask() = keyPaddingMask; + // model->Add(attnModule); + model->Add(tgtSeqLen, srcSeqLen, embedDim, nHeads, + attnMask, keyPaddingMask); + model->Add(embedDim * tgtSeqLen, vocabSize); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, batchSize); + model->Gradient(model->Parameters(), 0, gradient, batchSize); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + // MultiheadAttention* attnModule; + + arma::mat input, target, attnMask, keyPaddingMask; + const size_t tgtSeqLen; + const size_t srcSeqLen; + const size_t embedDim; + const size_t nHeads; + const size_t vocabSize; + const size_t batchSize; + } function; + + REQUIRE(CheckGradient(function) <= 3e-06); +} +*/ + +/** + * Simple tests for instance normalization layer. + * +TEST_CASE("InstanceNormLayerTest", "[ANNLayerTest]") +{ + arma::mat input, result, output, delta, deltaExpected; + arma::mat runningMean, runningVar; + + // Represents 2 images, each having 3 channels, and shape (3,2). + input << 1 << 19 << arma::endr + << 2 << 20 << arma::endr + << 3 << 21 << arma::endr + << 4 << 22 << arma::endr + << 5 << 23 << arma::endr + << 6 << 24 << arma::endr + << 7 << 25 << arma::endr + << 8 << 26 << arma::endr + << 9 << 27 << arma::endr + << 10 << 28 << arma::endr + << 11 << 29 << arma::endr + << 12 << 30 << arma::endr + << 13 << 31 << arma::endr + << 14 << 32 << arma::endr + << 15 << 33 << arma::endr + << 16 << 34 << arma::endr + << 17 << 35 << arma::endr + << 18 << 36 << arma::endr; + + // Output calculated using torch.nn.InstanceNorm2d(). + result << -1.4638 << -1.4638 << arma::endr + << -0.8783 << -0.8783 << arma::endr + << -0.2928 << -0.2928 << arma::endr + << 0.2928 << 0.2928 << arma::endr + << 0.8783 << 0.8783 << arma::endr + << 1.4638 << 1.4638 << arma::endr + << -1.4638 << -1.4638 << arma::endr + << -0.8783 << -0.8783 << arma::endr + << -0.2928 << -0.2928 << arma::endr + << 0.2928 << 0.2928 << arma::endr + << 0.8783 << 0.8783 << arma::endr + << 1.4638 << 1.4638 << arma::endr + << -1.4638 << -1.4638 << arma::endr + << -0.8783 << -0.8783 << arma::endr + << -0.2928 << -0.2928 << arma::endr + << 0.2928 << 0.2928 << arma::endr + << 0.8783 << 0.8783 << arma::endr + << 1.4638 << 1.4638 << arma::endr; + + // Calculated using torch.nn.InstanceNorm2d(). + deltaExpected << 1.8367 << 1.8367 << arma::endr + << 0.3967 << 0.3967 << arma::endr + << 0.0147 << 0.0147 << arma::endr + <<-0.0147 << -0.0147 << arma::endr + <<-0.3967 << -0.3967 << arma::endr + <<-1.8367 << -1.8367 << arma::endr + << 1.8367 << 1.8367 << arma::endr + << 0.3967 << 0.3967 << arma::endr + << 0.0147 << 0.0147 << arma::endr + <<-0.0147 << -0.0147 << arma::endr + <<-0.3967 << -0.3967 << arma::endr + <<-1.8367 << -1.8367 << arma::endr + << 1.8367 << 1.8367 << arma::endr + << 0.3967 << 0.3967 << arma::endr + << 0.0147 << 0.0147 << arma::endr + <<-0.0147 << -0.0147 << arma::endr + <<-0.3967 << -0.3967 << arma::endr + <<-1.8367 << -1.8367 << arma::endr; + + // Check Forward and Backward pass in non-deterministic mode. + InstanceNorm<> module(3, input.n_cols, 1e-5, false, 0.1); + output.zeros(arma::size(input)); + module.Forward(input, output); + CheckMatrices(output, result, 1e-1); + + module.Backward(input, output, delta); + CheckMatrices(delta, deltaExpected, 1e-1); + + runningMean = arma::mat(3, 1); + runningVar = arma::mat(3, 1); + runningMean(0) = 1.2500; + runningMean(1) = 1.8500; + runningMean(2) = 2.4500; + runningVar(0) = 1.2500; + runningVar(1) = 1.2500; + runningVar(2) = 1.2500; + + CheckMatrices(runningMean, module.TrainingMean(), 1e-1); + CheckMatrices(runningVar, module.TrainingVariance(), 1e-1); + + // Check Forward pass in deterministic mode. + InstanceNorm<> module1(3, input.n_cols, 1e-5, false, 0.1); + module1.Deterministic() = true; + output.zeros(arma::size(input)); + module1.Forward(input, output); + + // Calculated using torch.nn.InstanceNorm2d(). + result << 1.0000 << 18.9999 << arma::endr + << 2.0000 << 19.9999 << arma::endr + << 3.0000 << 20.9999 << arma::endr + << 4.0000 << 21.9999 << arma::endr + << 5.0000 << 22.9999 << arma::endr + << 6.0000 << 23.9999 << arma::endr + << 7.0000 << 24.9999 << arma::endr + << 8.0000 << 25.9999 << arma::endr + << 9.0000 << 26.9999 << arma::endr + << 10.0000 << 27.9999 << arma::endr + << 10.9999 << 28.9999 << arma::endr + << 11.9999 << 29.9999 << arma::endr + << 12.9999 << 30.9998 << arma::endr + << 13.9999 << 31.9998 << arma::endr + << 14.9999 << 32.9998 << arma::endr + << 15.9999 << 33.9998 << arma::endr + << 16.9999 << 34.9998 << arma::endr + << 17.9999 << 35.9998 << arma::endr; + + CheckMatrices(output, result, 1e-1); +} +*/ + +/** + * Test that the functions that can access the parameters of the + * Instance Norm layer work. + * +TEST_CASE("InstanceNormLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : size, eps. + InstanceNorm<> layer(7, 0, 1e-3); + + // Make sure we can get the parameters successfully. + REQUIRE(layer.InputSize() == 7); + REQUIRE(layer.Epsilon() == 1e-3); + + arma::mat runningMean(7, 1, arma::fill::randn); + arma::mat runningVariance(7, 1, arma::fill::randn); + + layer.TrainingVariance() = runningVariance; + layer.TrainingMean() = runningMean; + CheckMatrices(layer.TrainingVariance(), runningVariance); + CheckMatrices(layer.TrainingMean(), runningMean); +} +*/ + +/** + * Instance Norm layer numerical gradient test. + * +TEST_CASE("GradientInstanceNormLayerTest", "[ANNLayerTest]") +{ + // Add function gradient instantiation. + // To make this test robust, check it ten times. + bool pass = false; + for (size_t trial = 0; trial < 10; trial++) + { + struct GradientFunction + { + GradientFunction() + { + input = arma::randn(16, 1024); + arma::mat target; + target.ones(1, 1024); + + model = new FFN(); + model->ResetData(input, target); + model->Add >(); + model->Add >(1, 2, 3, 3, 1, 1, 0, 0, 4, 4); + model->Add > (2, 1024); + model->Add >(2 * 2 * 2, 2); + model->Add >(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1024, false); + model->Gradient(model->Parameters(), 0, gradient, 1024); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + double gradient = CheckGradient(function); + if (gradient < 1e-1) + { + pass = true; + break; + } + } + + REQUIRE(pass); +} +*/ diff --git a/src/mlpack/tests/ann/layer/concat.cpp b/src/mlpack/tests/ann/layer/concat.cpp new file mode 100644 index 0000000000..c20b07bef4 --- /dev/null +++ b/src/mlpack/tests/ann/layer/concat.cpp @@ -0,0 +1,227 @@ +/** + * @file tests/ann/layer/concat.cpp + * @author Marcus Edel + * @author Praveen Ch + * + * Tests the ann layer modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include +#include + +#include "../../test_catch_tools.hpp" +#include "../../catch.hpp" +#include "../../serialization.hpp" +#include "../ann_test_tools.hpp" + +using namespace mlpack; + +/** + * Simple concat module test. + */ +TEST_CASE("SimpleConcatLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta, error; + + Linear* moduleA = new Linear(10); + moduleA->InputDimensions() = std::vector({ 10 }); + moduleA->ComputeOutputDimensions(); + arma::mat weightsA(moduleA->WeightSize(), 1); + moduleA->SetWeights((double*) weightsA.memptr()); + moduleA->Parameters().randu(); + + Linear* moduleB = new Linear(10); + moduleB->InputDimensions() = std::vector({ 10 }); + moduleB->ComputeOutputDimensions(); + arma::mat weightsB(moduleB->WeightSize(), 1); + moduleB->SetWeights((double*) weightsB.memptr()); + moduleB->Parameters().randu(); + + Concat module; + module.Add(moduleA); + module.Add(moduleB); + module.InputDimensions() = std::vector({ 10 }); + module.ComputeOutputDimensions(); + + // Test the Forward function. + input = arma::zeros(10, 1); + output.set_size(module.OutputSize(), 1); + module.Forward(input, output); + + const double sumModuleA = arma::accu( + moduleA->Parameters().submat( + 100, 0, moduleA->Parameters().n_elem - 1, 0)); + const double sumModuleB = arma::accu( + moduleB->Parameters().submat( + 100, 0, moduleB->Parameters().n_elem - 1, 0)); + REQUIRE(sumModuleA + sumModuleB == + Approx(arma::accu(output.col(0))).epsilon(1e-5)); + + // Test the Backward function. + error = arma::zeros(20, 1); + delta.set_size(input.n_rows, input.n_cols); + module.Backward(input, error, delta); + REQUIRE(arma::accu(delta) == 0); +} + +/** + * Test to check Concat layer along different axes. + */ +TEST_CASE("ConcatAlongAxisTest", "[ANNLayerTest]") +{ + arma::mat output, input, error, outputA, outputB; + size_t inputWidth = 4, inputHeight = 4, inputChannel = 2; + size_t outputWidth, outputHeight, outputChannel = 2; + size_t kW = 3, kH = 3; + size_t batch = 1; + + // Using Convolution<> layer as inout to Concat<> layer. + // Compute the output shape of convolution layer. + outputWidth = (inputWidth - kW) + 1; + outputHeight = (inputHeight - kH) + 1; + + input = arma::ones(inputWidth * inputHeight * inputChannel, batch); + + Convolution* moduleA = new Convolution(outputChannel, kW, kH, 1, 1, 0, 0); + Convolution* moduleB = new Convolution(outputChannel, kW, kH, 1, 1, 0, 0); + + moduleA->InputDimensions() = std::vector({ inputWidth, inputHeight }); + moduleA->ComputeOutputDimensions(); + arma::mat weightsA(moduleA->WeightSize(), 1); + moduleA->SetWeights((double*) weightsA.memptr()); + moduleA->Parameters().randu(); + + moduleB->InputDimensions() = std::vector({ inputWidth, inputHeight }); + moduleB->ComputeOutputDimensions(); + arma::mat weightsB(moduleB->WeightSize(), 1); + moduleB->SetWeights((double*) weightsB.memptr()); + moduleB->Parameters().randu(); + + // Compute output of each layer. + outputA.set_size(moduleA->OutputSize(), 1); + outputB.set_size(moduleB->OutputSize(), 1); + moduleA->Forward(input, outputA); + moduleB->Forward(input, outputB); + + arma::cube A(outputA.memptr(), outputWidth, outputHeight, outputChannel); + arma::cube B(outputB.memptr(), outputWidth, outputHeight, outputChannel); + + error = arma::ones(outputWidth * outputHeight * outputChannel * 2, 1); + + for (size_t axis = 0; axis < 3; ++axis) + { + size_t x = 1, y = 1, z = 1; + arma::cube calculatedOut; + if (axis == 0) + { + calculatedOut.set_size(2 * outputWidth, outputHeight, outputChannel); + for (size_t i = 0; i < A.n_slices; ++i) + { + arma::mat aMat = A.slice(i); + arma::mat bMat = B.slice(i); + calculatedOut.slice(i) = arma::join_cols(aMat, bMat); + } + x = 2; + } + if (axis == 1) + { + calculatedOut.set_size(outputWidth, 2 * outputHeight, outputChannel); + for (size_t i = 0; i < A.n_slices; ++i) + { + arma::mat aMat = A.slice(i); + arma::mat bMat = B.slice(i); + calculatedOut.slice(i) = arma::join_rows(aMat, bMat); + } + y = 2; + } + if (axis == 2) + { + calculatedOut = arma::join_slices(A, B); + z = 2; + } + + // Compute output of Concat<> layer. + Concat module(axis); + module.Add(moduleA); + module.Add(moduleB); + module.InputDimensions() = std::vector({ inputWidth, inputHeight }); + module.ComputeOutputDimensions(); + output.set_size(module.OutputSize(), 1); + module.Forward(input, output); + arma::cube concatOut(output.memptr(), x * outputWidth, + y * outputHeight, z * outputChannel); + + // Verify if the output reshaped to cubes are similar. + CheckMatrices(concatOut, calculatedOut, 1e-12); + + // Ensure that the child layers don't get deleted when `module` is + // deallocated. + module.Network().clear(); + } + + delete moduleA; + delete moduleB; +} + +/** + * Test that the function that can access the axis parameter of the + * Concat layer works. + */ +TEST_CASE("ConcatLayerParametersTest", "[ANNLayerTest]") +{ + Concat layer(2); + + // Make sure we can get the parameters successfully. + REQUIRE(layer.Axis() == 2); +} + +/** + * Concat layer numerical gradient test. + */ +TEST_CASE("GradientConcatLayerTest", "[ANNLayerTest]") +{ + // Concat function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(10, 1)), + target(arma::mat("0")) + { + model = new FFN(); + model->ResetData(input, target); + model->Add(10); + + concat = new Concat(); + concat->Add(5); + concat->Add(5); + model->Add(concat); + model->Add(2); + + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + Concat* concat; + arma::mat input, target; + } function; + + REQUIRE(CheckGradient(function) <= 1e-4); +} diff --git a/src/mlpack/tests/ann/layer/concatenate.cpp b/src/mlpack/tests/ann/layer/concatenate.cpp new file mode 100644 index 0000000000..a4fde0733b --- /dev/null +++ b/src/mlpack/tests/ann/layer/concatenate.cpp @@ -0,0 +1,95 @@ +/** + * @file tests/ann/layer/concatenate.cpp + * @author Marcus Edel + * @author Praveen Ch + * + * Tests the ann layer modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include +#include + +#include "../../test_catch_tools.hpp" +#include "../../catch.hpp" +#include "../../serialization.hpp" +#include "../ann_test_tools.hpp" + +using namespace mlpack; + + +/** + * Simple concatenate module test. + */ +TEST_CASE("SimpleConcatenateLayerTest", "[ANNLayerTest]") +{ + arma::mat input = arma::ones(5, 1); + arma::mat output, delta; + + Concatenate module; + module.Concat() = arma::ones(5, 1) * 0.5; + module.InputDimensions() = std::vector({ 5 }); + module.ComputeOutputDimensions(); + + // Test the Forward function. + output.set_size(module.OutputSize(), 1); + module.Forward(input, output); + + REQUIRE(arma::accu(output) == 7.5); + + // Test the Backward function. + delta.set_size(5, 1); + module.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 5); +} + +/** + * Concatenate layer numerical gradient test. + */ +TEST_CASE("GradientConcatenateLayerTest", "[ANNLayerTest]") +{ + // Concatenate function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(10, 1)), + target(arma::mat("0")) + { + model = new FFN(); + model->ResetData(input, target); + model->Add(5); + + arma::mat concat = arma::ones(5, 1); + // concatenate = new Concatenate(); + // concatenate->Concat() = concat; + // model->Add(concatenate); + model->Add(concat); + + model->Add(5); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + Concatenate* concatenate; + arma::mat input, target; + } function; + + REQUIRE(CheckGradient(function) <= 1e-4); +} \ No newline at end of file diff --git a/src/mlpack/tests/ann/layer_test.cpp b/src/mlpack/tests/ann/layer_test.cpp index e22bf8573f..2389431e19 100644 --- a/src/mlpack/tests/ann/layer_test.cpp +++ b/src/mlpack/tests/ann/layer_test.cpp @@ -22,6 +22,8 @@ #include "layer/alpha_dropout.cpp" #include "layer/batch_norm.cpp" #include "layer/convolution.cpp" +#include "layer/concat.cpp" +#include "layer/concatenate.cpp" #include "layer/dropout.cpp" #include "layer/grouped_convolution.cpp" #include "layer/identity.cpp" diff --git a/src/mlpack/tests/ann/loss_functions_test.cpp b/src/mlpack/tests/ann/loss_functions_test.cpp index 9c6695eef0..2afa26fe3a 100644 --- a/src/mlpack/tests/ann/loss_functions_test.cpp +++ b/src/mlpack/tests/ann/loss_functions_test.cpp @@ -1251,3 +1251,24 @@ TEST_CASE("NegativeLogLikelihoodLossTest", "[LossFunctionsTest]") REQUIRE(output.n_cols == input.n_cols); CheckMatrices(output, expectedOutput, 0.1); } + +/** + * Jacobian negative log likelihood module test. + */ +TEST_CASE("JacobianNegativeLogLikelihoodLayerTest", "[LossFunctionsTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + NegativeLogLikelihood module; + const size_t inputElements = RandInt(5, 100); + arma::mat input; + RandomInitialization init(0, 1); + init.Initialize(input, inputElements, 1); + + arma::mat target(1, 1); + target(0) = RandInt(0, inputElements - 2); + + double error = JacobianPerformanceTest(module, input, target); + REQUIRE(error <= 1e-5); + } +} From 95b04c78ae6addd2c077500948c25c82ecb11f77 Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Mon, 27 Feb 2023 00:53:53 +0530 Subject: [PATCH 81/88] style fix --- src/mlpack/tests/ann/layer/concatenate.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/ann/layer/concatenate.cpp b/src/mlpack/tests/ann/layer/concatenate.cpp index a4fde0733b..0cbcb3ef2b 100644 --- a/src/mlpack/tests/ann/layer/concatenate.cpp +++ b/src/mlpack/tests/ann/layer/concatenate.cpp @@ -92,4 +92,4 @@ TEST_CASE("GradientConcatenateLayerTest", "[ANNLayerTest]") } function; REQUIRE(CheckGradient(function) <= 1e-4); -} \ No newline at end of file +} From 14c81ecf4b9e6366fdce0149bd3a6ecb1ef3be43 Mon Sep 17 00:00:00 2001 From: AdarshSantoria Date: Mon, 27 Feb 2023 00:58:33 +0530 Subject: [PATCH 82/88] fix file --- src/mlpack/tests/ann/layer/ann_layer_test.cpp | 4538 ----------------- .../tests/ann/not_adapted/ann_layer_test.cpp | 512 -- 2 files changed, 5050 deletions(-) delete mode 100644 src/mlpack/tests/ann/layer/ann_layer_test.cpp diff --git a/src/mlpack/tests/ann/layer/ann_layer_test.cpp b/src/mlpack/tests/ann/layer/ann_layer_test.cpp deleted file mode 100644 index cfd9ec0464..0000000000 --- a/src/mlpack/tests/ann/layer/ann_layer_test.cpp +++ /dev/null @@ -1,4538 +0,0 @@ -/** - * @file tests/ann_layer_test.cpp - * @author Marcus Edel - * @author Praveen Ch - * - * Tests the ann layer modules. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include - -#include "../test_catch_tools.hpp" -#include "../catch.hpp" -#include "../serialization.hpp" -#include "ann_test_tools.hpp" - -using namespace mlpack; - -// // network1 should be allocated with `new`, and trained on some data. -// template -// void CheckRNNCopyFunction(ModelType* network1, -// MatType& trainData, -// MatType& trainLabels, -// const size_t maxEpochs) -// { -// arma::cube predictions1; -// arma::cube predictions2; -// ens::StandardSGD opt(0.1, 1, maxEpochs * trainData.n_slices, -100, false); - -// network1->Train(trainData, trainLabels, opt); -// network1->Predict(trainData, predictions1); - -// RNN<> network2 = *network1; -// delete network1; - -// // Deallocating all of network1's memory, so that network2 does not use any -// // of that memory. -// network2.Predict(trainData, predictions2); -// CheckMatrices(predictions1, predictions2); -// } - -// // network1 should be allocated with `new`, and trained on some data. -// template -// void CheckRNNMoveFunction(ModelType* network1, -// MatType& trainData, -// MatType& trainLabels, -// const size_t maxEpochs) -// { -// arma::cube predictions1; -// arma::cube predictions2; -// ens::StandardSGD opt(0.1, 1, maxEpochs * trainData.n_slices, -100, false); - -// network1->Train(trainData, trainLabels, opt); -// network1->Predict(trainData, predictions1); - -// RNN<> network2(std::move(*network1)); -// delete network1; - -// // Deallocating all of network1's memory, so that network2 does not use any -// // of that memory. -// network2.Predict(trainData, predictions2); -// CheckMatrices(predictions1, predictions2); -// } - -/** - * Simple add module test. - * -TEST_CASE("SimpleAddLayerTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta; - Add module(10); - module.Parameters().randu(); - - // Test the Forward function. - input = arma::zeros(10, 1); - module.Forward(input, output); - REQUIRE(arma::accu(module.Parameters()) == arma::accu(output)); - - // Test the Backward function. - module.Backward(input, output, delta); - REQUIRE(arma::accu(output) == arma::accu(delta)); - - // Test the forward function. - input = arma::ones(10, 1); - module.Forward(input, output); - REQUIRE(10 + arma::accu(module.Parameters()) == - Approx(arma::accu(output)).epsilon(1e-5)); - - // Test the backward function. - module.Backward(input, output, delta); - REQUIRE(arma::accu(output) == Approx(arma::accu(delta)).epsilon(1e-5)); -} -*/ - -/** - * Jacobian add module test. - * -TEST_CASE("JacobianAddLayerTest", "[ANNLayerTest]") -{ - for (size_t i = 0; i < 5; ++i) - { - const size_t elements = RandInt(2, 1000); - arma::mat input; - input.set_size(elements, 1); - - Add module(elements); - module.Parameters().randu(); - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -} -*/ - -/** - * Add layer numerical gradient test. - * -TEST_CASE("GradientAddLayerTest", "[ANNLayerTest]") -{ - // Add function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - input(arma::randu(10, 1)), - target(arma::mat("0")) - { - model = new FFN(); - model->ResetData(input, target); - model->Add(); - model->Add(10, 10); - model->Add(10); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - REQUIRE(CheckGradient(function) <= 1e-4); -}*/ - -/** - * Test that the function that can access the outSize parameter of - * the Add layer works. - * -TEST_CASE("AddLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter : outSize. - Add layer(7); - - // Make sure we can get the parameter successfully. - REQUIRE(layer.OutputSize() == 7); -}*/ - -/** - * Simple constant module test. - * -TEST_CASE("SimpleConstantLayerTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta; - Constant module(10, 3.0); - - // Test the Forward function. - input = arma::zeros(10, 1); - module.Forward(input, output); - REQUIRE(arma::accu(output) == 30.0); - - // Test the Backward function. - module.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 0); - - // Test the forward function. - input = arma::ones(10, 1); - module.Forward(input, output); - REQUIRE(arma::accu(output) == 30.0); - - // Test the backward function. - module.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 0); -}*/ - -/** - * Jacobian constant module test. - * -TEST_CASE("JacobianConstantLayerTest", "[ANNLayerTest]") -{ - for (size_t i = 0; i < 5; ++i) - { - const size_t elements = RandInt(2, 1000); - arma::mat input; - input.set_size(elements, 1); - - Constant module(elements, 1.0); - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -}*/ - -/** - * Test that the function that can access the outSize parameter of the - * Constant layer works. - * -TEST_CASE("ConstantLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter : outSize. - Constant layer(7); - - // Make sure we can get the parameter successfully. - REQUIRE(layer.OutSize() == 7); -}*/ - -// /** -// * Simple linear module test. -// */ -// TEST_CASE("SimpleLinearLayerTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta; -// Linear<> module(10, 10); -// module.Parameters().randu(); -// module.Reset(); - -// // Test the Forward function. -// input = arma::zeros(10, 1); -// module.Forward(input, output); -// REQUIRE(arma::accu(module.Parameters().submat(100, -// 0, module.Parameters().n_elem - 1, 0)) == -// Approx(arma::accu(output)).epsilon(1e-5)); - -// // Test the Backward function. -// module.Backward(input, input, delta); -// REQUIRE(arma::accu(delta) == 0); -// } - -// /** -// * Jacobian linear module test. -// */ -// TEST_CASE("JacobianLinearLayerTest", "[ANNLayerTest]") -// { -// for (size_t i = 0; i < 5; ++i) -// { -// const size_t inputElements = RandInt(2, 1000); -// const size_t outputElements = RandInt(2, 1000); - -// arma::mat input; -// input.set_size(inputElements, 1); - -// Linear<> module(inputElements, outputElements); -// module.Parameters().randu(); - -// double error = JacobianTest(module, input); -// REQUIRE(error <= 1e-5); -// } -// } - -// /** -// * Linear layer numerical gradient test. -// */ -// TEST_CASE("GradientLinearLayerTest", "[ANNLayerTest]") -// { -// // Linear function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(10, 1)), -// target(arma::mat("1")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(10, 10); -// model->Add >(10, 2); -// model->Add >(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// arma::mat input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -// /** -// * Simple noisy linear module test. -// */ -// TEST_CASE("SimpleNoisyLinearLayerTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta; -// NoisyLinear<> module(10, 10); -// module.Parameters().randu(); -// module.Reset(); - -// // Test the Backward function. -// module.Backward(input, input, delta); -// REQUIRE(arma::accu(delta) == 0); -// } - -// /** -// * Jacobian noisy linear module test. -// */ -// TEST_CASE("JacobianNoisyLinearLayerTest", "[ANNLayerTest]") -// { -// const size_t inputElements = RandInt(2, 1000); -// const size_t outputElements = RandInt(2, 1000); - -// arma::mat input; -// input.set_size(inputElements, 1); - -// NoisyLinear<> module(inputElements, outputElements); -// module.Parameters().randu(); - -// double error = JacobianTest(module, input); -// REQUIRE(error <= 1e-5); -// } - -// /** -// * Noisy Linear layer numerical gradient test. -// */ -// TEST_CASE("GradientNoisyLinearLayerTest", "[ANNLayerTest]") -// { -// // Noisy linear function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(10, 1)), -// target(arma::mat("1")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(10, 10); -// model->Add >(10, 2); -// model->Add >(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// arma::mat input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -/** - * Jacobian LeakyReLU module test. - * -TEST_CASE("JacobianLeakyReLULayerTest", "[ANNLayerTest]") -{ - for (size_t i = 0; i < 5; ++i) - { - const size_t inputElements = RandInt(2, 1000); - - arma::mat input; - input.set_size(inputElements, 1); - - LeakyReLU module; - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -} -*/ - -/** - * Jacobian FlexibleReLU module test. - * -TEST_CASE("JacobianFlexibleReLULayerTest", "[ANNLayerTest]") -{ - for (size_t i = 0; i < 5; ++i) - { - const size_t inputElements = RandInt(2, 1000); - - arma::mat input; - input.set_size(inputElements, 1); - - FlexibleReLU module; - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -} -*/ - -/** - * Flexible ReLU layer numerical gradient test. - * -TEST_CASE("GradientFlexibleReLULayerTest", "[ANNLayerTest]") -{ - // Add function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - input(arma::randu(2, 1)), - target(arma::mat("0")) - { - model = new FFN( - NegativeLogLikelihood(), RandomInitialization(0.1, 0.5)); - - model->ResetData(input, target); - model->Add(2, 2); - model->Add(2, 5); - model->Add(0.05); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - REQUIRE(CheckGradient(function) <= 1e-4); -} -*/ - -/** - * Jacobian MultiplyConstant module test. - * -TEST_CASE("JacobianMultiplyConstantLayerTest", "[ANNLayerTest]") -{ - for (size_t i = 0; i < 5; ++i) - { - const size_t inputElements = RandInt(2, 1000); - - arma::mat input; - input.set_size(inputElements, 1); - - MultiplyConstant module(3.0); - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -} -*/ - -/** - * Check whether copying and moving network with MultiplyConstant is working or - * not. - */ -// TEST_CASE("CheckCopyMoveMultiplyConstantTest", "[ANNLayerTest]") -// { -// arma::mat input(2, 1000); -// input.randu(); -// -// arma::mat output1; -// arma::mat output2; -// arma::mat output3; -// arma::mat output4; -// -// MultiplyConstant<> *module1 = new MultiplyConstant<>(3.0); -// module1->Forward(input, output1); -// -// MultiplyConstant<> module2 = *module1; -// delete module1; -// -// module2.Forward(input, output2); -// CheckMatrices(output1, output2); -// -// MultiplyConstant<> *module3 = new MultiplyConstant<>(3.0); -// module3->Forward(input, output3); -// -// MultiplyConstant<> module4(std::move(*module3)); -// delete module3; -// -// module4.Forward(input, output4); -// CheckMatrices(output3, output4); -// } - -/** - * Jacobian HardTanH module test. - * -TEST_CASE("JacobianHardTanHLayerTest", "[ANNLayerTest]") -{ - for (size_t i = 0; i < 5; ++i) - { - const size_t inputElements = RandInt(2, 1000); - - arma::mat input; - input.set_size(inputElements, 1); - - HardTanH module; - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -} -*/ - -/** - * Simple select module test. - * -TEST_CASE("SimpleSelectLayerTest", "[ANNLayerTest]") -{ - // TODO: this needs to be adapted - arma::mat outputA, outputB, input, delta; - - input = arma::ones(10, 5); - for (size_t i = 0; i < input.n_cols; ++i) - { - input.col(i) *= i; - } - - // Test the Forward function. - Select moduleA(3); - moduleA.Forward(input, outputA); - REQUIRE(30 == arma::accu(outputA)); - - // Test the Forward function. - Select moduleB(3, 5); - moduleB.Forward(input, outputB); - REQUIRE(15 == arma::accu(outputB)); - - // Test the Backward function. - moduleA.Backward(input, outputA, delta); - REQUIRE(30 == arma::accu(delta)); - - // Test the Backward function. - moduleB.Backward(input, outputA, delta); - REQUIRE(15 == arma::accu(delta)); -} -*/ - -/** - * Test that the functions that can access the parameters of the - * Select layer work. - * -TEST_CASE("SelectLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : index, elements. - Select layer(3, 5); - - // Make sure we can get the parameters successfully. - REQUIRE(layer.Index() == 3); - REQUIRE(layer.NumElements() == 5); -} -*/ - -/** - * Simple join module test. - * -TEST_CASE("SimpleJoinLayerTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta; - input = arma::ones(10, 5); - - // Test the Forward function. - Join module; - module.Forward(input, output); - REQUIRE(50 == arma::accu(output)); - - bool b = output.n_rows == 1 || output.n_cols == 1; - REQUIRE(b == true); - - // Test the Backward function. - module.Backward(input, output, delta); - REQUIRE(50 == arma::accu(delta)); - - b = delta.n_rows == input.n_rows && input.n_cols; - REQUIRE(b == true); -} -*/ - -// /** -// * Simple add merge module test. -// */ -// TEST_CASE("SimpleAddMergeLayerTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta; -// input = arma::ones(10, 1); - -// for (size_t i = 0; i < 5; ++i) -// { -// AddMerge<> module(false, false); -// const size_t numMergeModules = RandInt(2, 10); -// for (size_t m = 0; m < numMergeModules; ++m) -// { -// IdentityLayer<> identityLayer; -// identityLayer.Forward(input, identityLayer.OutputParameter()); - -// module.Add >(identityLayer); -// } - -// // Test the Forward function. -// module.Forward(input, output); -// REQUIRE(10 * numMergeModules == arma::accu(output)); - -// // Test the Backward function. -// module.Backward(input, output, delta); -// REQUIRE(arma::accu(output) == arma::accu(delta)); -// } -// } - -// /** -// * Test the LSTM layer with a user defined rho parameter and without. -// */ -// TEST_CASE("LSTMRrhoTest", "[ANNLayerTest]") -// { -// const size_t rho = 5; -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::ones(1, 1, 5); -// RandomInitialization init(0.5, 0.5); - -// // Create model with user defined rho parameter. -// RNN modelA( -// rho, false, NegativeLogLikelihood(), init); -// modelA.Add >(); -// modelA.Add >(1, 10); - -// // Use LSTM layer with rho. -// modelA.Add >(10, 3, rho); -// modelA.Add >(); - -// // Create model without user defined rho parameter. -// RNN modelB( -// rho, false, NegativeLogLikelihood(), init); -// modelB.Add >(); -// modelB.Add >(1, 10); - -// // Use LSTM layer with rho = MAXSIZE. -// modelB.Add >(10, 3); -// modelB.Add >(); - -// ens::StandardSGD opt(0.1, 1, 5, -100, false); -// modelA.Train(input, target, opt); -// modelB.Train(input, target, opt); - -// CheckMatrices(modelB.Parameters(), modelA.Parameters()); -// } - -// /** -// * LSTM layer numerical gradient test. -// */ -// TEST_CASE("GradientLSTMLayerTest", "[ANNLayerTest]") -// { -// // LSTM function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(1, 1, 5)), -// target(arma::ones(1, 1, 5)) -// { -// const size_t rho = 5; - -// model = new RNN(rho); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 10); -// model->Add >(10, 3, rho); -// model->Add >(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// RNN* model; -// arma::cube input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -// /** -// * Test that the functions that can modify and access the parameters of the -// * LSTM layer work. -// */ -// TEST_CASE("LSTMLayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : inSize, outSize, rho. -// LSTM<> layer1(1, 2, 3); -// LSTM<> layer2(1, 2, 4); - -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InSize() == 1); -// REQUIRE(layer1.OutSize() == 2); -// REQUIRE(layer1.Rho() == 3); - -// // Now modify the parameters to match the second layer. -// layer1.Rho() = 4; - -// // Now ensure all the results are the same. -// REQUIRE(layer1.InSize() == layer2.InSize()); -// REQUIRE(layer1.OutSize() == layer2.OutSize()); -// REQUIRE(layer1.Rho() == layer2.Rho()); -// } - -// /** -// * Test the FastLSTM layer with a user defined rho parameter and without. -// */ -// TEST_CASE("FastLSTMRrhoTest", "[ANNLayerTest]") -// { -// const size_t rho = 5; -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::ones(1, 1, 5); -// RandomInitialization init(0.5, 0.5); - -// // Create model with user defined rho parameter. -// RNN modelA( -// rho, false, NegativeLogLikelihood(), init); -// modelA.Add >(); -// modelA.Add >(1, 10); - -// // Use FastLSTM layer with rho. -// modelA.Add >(10, 3, rho); -// modelA.Add >(); - -// // Create model without user defined rho parameter. -// RNN modelB( -// rho, false, NegativeLogLikelihood(), init); -// modelB.Add >(); -// modelB.Add >(1, 10); - -// // Use FastLSTM layer with rho = MAXSIZE. -// modelB.Add >(10, 3); -// modelB.Add >(); - -// ens::StandardSGD opt(0.1, 1, 5, -100, false); -// modelA.Train(input, target, opt); -// modelB.Train(input, target, opt); - -// CheckMatrices(modelB.Parameters(), modelA.Parameters()); -// } - -// /** -// * FastLSTM layer numerical gradient test. -// */ -// TEST_CASE("GradientFastLSTMLayerTest", "[ANNLayerTest]") -// { -// // Fast LSTM function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(1, 1, 5)), -// target(arma::ones(1, 1, 5)) -// { -// const size_t rho = 5; - -// model = new RNN(rho); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 10); -// model->Add >(10, 3, rho); -// model->Add >(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// RNN* model; -// arma::cube input, target; -// } function; - -// // The threshold should be << 0.1 but since the Fast LSTM layer uses an -// // approximation of the sigmoid function the estimated gradient is not -// // correct. -// REQUIRE(CheckGradient(function) <= 0.2); -// } - -// /** -// * Test that the functions that can modify and access the parameters of the -// * Fast LSTM layer work. -// */ -// TEST_CASE("FastLSTMLayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : inSize, outSize, rho. -// FastLSTM<> layer1(1, 2, 3); -// FastLSTM<> layer2(1, 2, 4); - -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InSize() == 1); -// REQUIRE(layer1.OutSize() == 2); -// REQUIRE(layer1.Rho() == 3); - -// // Now modify the parameters to match the second layer. -// layer1.Rho() = 4; - -// // Now ensure all the results are the same. -// REQUIRE(layer1.InSize() == layer2.InSize()); -// REQUIRE(layer1.OutSize() == layer2.OutSize()); -// REQUIRE(layer1.Rho() == layer2.Rho()); -// } - -// /** -// * Check whether copying and moving network with FastLSTM is working or not. -// */ -// TEST_CASE("CheckCopyMoveFastLSTMTest", "[ANNLayerTest]") -// { -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::ones(1, 1, 5); -// const size_t rho = 5; - -// RNN *model1 = -// new RNN(rho); -// model1->ResetData(input, target); -// model1->Add >(); -// model1->Add >(1, 10); -// model1->Add >(10, 3, rho); -// model1->Add >(); - -// RNN *model2 = -// new RNN(rho); -// model2->ResetData(input, target); -// model2->Add >(); -// model2->Add >(1, 10); -// model2->Add >(10, 3, rho); -// model2->Add >(); - -// // Check whether copy constructor is working or not. -// CheckRNNCopyFunction<>(model1, input, target, 1); - -// // Check whether move constructor is working or not. -// CheckRNNMoveFunction<>(model2, input, target, 1); -// } - -// /** -// * Check whether copying and moving network with LSTM is working or not. -// */ -// TEST_CASE("CheckCopyMoveLSTMTest", "[ANNLayerTest]") -// { -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::ones(1, 1, 5); -// const size_t rho = 5; - -// RNN *model1 = -// new RNN(rho); -// model1->ResetData(input, target); -// model1->Add >(); -// model1->Add >(1, 10); -// model1->Add >(10, 3, rho); -// model1->Add >(); - -// RNN *model2 = -// new RNN(rho); -// model2->ResetData(input, target); -// model2->Add >(); -// model2->Add >(1, 10); -// model2->Add >(10, 3, rho); -// model2->Add >(); - -// // Check whether copy constructor is working or not. -// CheckRNNCopyFunction<>(model1, input, target, 1); - -// // Check whether move constructor is working or not. -// CheckRNNMoveFunction<>(model2, input, target, 1); -// } - -// /** -// * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell -// * state. Besides output, the overloaded function provides read access to cell -// * state of the LSTM layer. -// */ -// TEST_CASE("ReadCellStateParamLSTMLayerTest", "[ANNLayerTest]") -// { -// const size_t rho = 5, inputSize = 3, outputSize = 2; - -// // Provide input of all ones. -// arma::cube input = arma::ones(inputSize, outputSize, rho); - -// arma::mat inputGate, forgetGate, outputGate, hidden; -// arma::mat outLstm, cellLstm; - -// // LSTM layer. -// LSTM<> lstm(inputSize, outputSize, rho); -// lstm.Reset(); -// lstm.ResetCell(rho); - -// // Initialize the weights to all ones. -// lstm.Parameters().ones(); - -// arma::mat inputWeight = arma::ones(outputSize, inputSize); -// arma::mat outputWeight = arma::ones(outputSize, outputSize); -// arma::mat bias = arma::ones(outputSize, input.n_cols); -// arma::mat cellCalc = arma::zeros(outputSize, input.n_cols); -// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); - -// for (size_t seqNum = 0; seqNum < rho; ++seqNum) -// { -// // Wrap a matrix around our data to avoid a copy. -// arma::mat stepData(input.slice(seqNum).memptr(), -// input.n_rows, input.n_cols, false, true); - -// // Apply Forward() on LSTM layer. -// lstm.Forward(stepData, // Input. -// outLstm, // Output. -// cellLstm, // Cell state. -// false); // Don't write into the cell state. - -// // Compute the value of cell state and output. -// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); - -// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); - -// // z = tanh(W.dot(x) + W.dot(h) + b). -// hidden = arma::tanh(inputWeight * stepData + -// outputWeight * outCalc + bias); - -// // c = f * c + i * z. -// cellCalc = forgetGate % cellCalc + inputGate % hidden; - -// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); - -// // h = o * tanh(c). -// outCalc = outputGate % arma::tanh(cellCalc); - -// CheckMatrices(outLstm, outCalc, 1e-12); -// CheckMatrices(cellLstm, cellCalc, 1e-12); -// } -// } - -// /** -// * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell -// * state. Besides output, the overloaded function provides write access to cell -// * state of the LSTM layer. -// */ -// TEST_CASE("WriteCellStateParamLSTMLayerTest", "[ANNLayerTest]") -// { -// const size_t rho = 5, inputSize = 3, outputSize = 2; - -// // Provide input of all ones. -// arma::cube input = arma::ones(inputSize, outputSize, rho); - -// arma::mat inputGate, forgetGate, outputGate, hidden; -// arma::mat outLstm, cellLstm; -// arma::mat cellCalc; - -// // LSTM layer. -// LSTM<> lstm(inputSize, outputSize, rho); -// lstm.Reset(); -// lstm.ResetCell(rho); - -// // Initialize the weights to all ones. -// lstm.Parameters().ones(); - -// arma::mat inputWeight = arma::ones(outputSize, inputSize); -// arma::mat outputWeight = arma::ones(outputSize, outputSize); -// arma::mat bias = arma::ones(outputSize, input.n_cols); -// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); - -// for (size_t seqNum = 0; seqNum < rho; ++seqNum) -// { -// // Wrap a matrix around our data to avoid a copy. -// arma::mat stepData(input.slice(seqNum).memptr(), -// input.n_rows, input.n_cols, false, true); - -// if (cellLstm.is_empty()) -// { -// // Set the cell state to zeros. -// cellLstm = arma::zeros(outputSize, input.n_cols); -// cellCalc = arma::zeros(outputSize, input.n_cols); -// } -// else -// { -// // Set the cell state to zeros. -// cellLstm = arma::zeros(cellLstm.n_rows, cellLstm.n_cols); -// cellCalc = arma::zeros(cellCalc.n_rows, cellCalc.n_cols); -// } - -// // Apply Forward() on the LSTM layer. -// lstm.Forward(stepData, // Input. -// outLstm, // Output. -// cellLstm, // Cell state. -// true); // Write into cell state. - -// // Compute the value of cell state and output. -// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); - -// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); - -// // z = tanh(W.dot(x) + W.dot(h) + b). -// hidden = arma::tanh(inputWeight * stepData + -// outputWeight * outCalc + bias); - -// // c = f * c + i * z. -// cellCalc = forgetGate % cellCalc + inputGate % hidden; - -// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); - -// // h = o * tanh(c). -// outCalc = outputGate % arma::tanh(cellCalc); - -// CheckMatrices(outLstm, outCalc, 1e-12); -// CheckMatrices(cellLstm, cellCalc, 1e-12); -// } - -// // Attempting to write empty matrix into cell state. -// lstm.Reset(); -// lstm.ResetCell(rho); -// arma::mat stepData(input.slice(0).memptr(), -// input.n_rows, input.n_cols, false, true); - -// lstm.Forward(stepData, // Input. -// outLstm, // Output. -// cellLstm, // Cell state. -// true); // Write into cell state. - -// for (size_t seqNum = 1; seqNum < rho; ++seqNum) -// { -// arma::mat empty; -// // Should throw error. -// REQUIRE_THROWS_AS(lstm.Forward(stepData, // Input. -// outLstm, // Output. -// empty, // Cell state. -// true), // Write into cell state. -// std::runtime_error); -// } -// } - -// /** -// * Test that the functions that can modify and access the parameters of the -// * GRU layer work. -// */ -// TEST_CASE("GRULayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : inSize, outSize, rho. -// GRU<> layer1(1, 2, 3); -// GRU<> layer2(1, 2, 4); - -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InSize() == 1); -// REQUIRE(layer1.OutSize() == 2); -// REQUIRE(layer1.Rho() == 3); - -// // Now modify the parameters to match the second layer. -// layer1.Rho() = 4; - -// // Now ensure all the results are the same. -// REQUIRE(layer1.InSize() == layer2.InSize()); -// REQUIRE(layer1.OutSize() == layer2.OutSize()); -// REQUIRE(layer1.Rho() == layer2.Rho()); -// } - -// /** -// * Check if the gradients computed by GRU cell are close enough to the -// * approximation of the gradients. -// */ -// TEST_CASE("GradientGRULayerTest", "[ANNLayerTest]") -// { -// // GRU function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(1, 1, 5)), -// target(arma::ones(1, 1, 5)) -// { -// const size_t rho = 5; - -// model = new RNN(rho); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 10); -// model->Add >(10, 3, rho); -// model->Add >(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// arma::mat output; -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// RNN* model; -// arma::cube input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -// /** -// * GRU layer manual forward test. -// */ -// TEST_CASE("ForwardGRULayerTest", "[ANNLayerTest]") -// { -// // This will make it easier to clean memory later. -// GRU<>* gruAlloc = new GRU<>(3, 3, 5); -// GRU<>& gru = *gruAlloc; - -// // Initialize the weights to all ones. -// NetworkInitialization -// networkInit(ConstInitialization(1)); -// networkInit.Initialize(gru.Model(), gru.Parameters()); - -// // Provide input of all ones. -// arma::mat input = arma::ones(3, 1); -// arma::mat output; - -// gru.Forward(input, output); - -// // Compute the z_t gate output. -// arma::mat expectedOutput = arma::ones(3, 1); -// expectedOutput *= -4; -// expectedOutput = arma::exp(expectedOutput); -// expectedOutput = arma::ones(3, 1) / (arma::ones(3, 1) + expectedOutput); -// expectedOutput = (arma::ones(3, 1) - expectedOutput) % expectedOutput; - -// // For the first input the output should be equal to the output of -// // gate z_t as the previous output fed to the cell is all zeros. -// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); - -// expectedOutput = output; - -// gru.Forward(input, output); - -// double s = arma::as_scalar(arma::sum(expectedOutput)); - -// // Compute the value of z_t gate for the second input. -// arma::mat z_t = arma::ones(3, 1); -// z_t *= -(s + 4); -// z_t = arma::exp(z_t); -// z_t = arma::ones(3, 1) / (arma::ones(3, 1) + z_t); - -// // Compute the value of o_t gate for the second input. -// arma::mat o_t = arma::ones(3, 1); -// o_t *= -(arma::as_scalar(arma::sum(expectedOutput % z_t)) + 4); -// o_t = arma::exp(o_t); -// o_t = arma::ones(3, 1) / (arma::ones(3, 1) + o_t); - -// // Expected output for the second input. -// expectedOutput = z_t % expectedOutput + (arma::ones(3, 1) - z_t) % o_t; - -// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); - -// LayerTypes<> layer(gruAlloc); -// boost::apply_visitor(DeleteVisitor(), layer); -// } - -/** - * Simple add merge module test. - */ -// TEST_CASE("SimpleAddMergeLayerTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta; -// input = arma::ones(10, 1); -// -// for (size_t i = 0; i < 5; ++i) -// { -// AddMerge<> module(false, false); -// const size_t numMergeModules = RandInt(2, 10); -// for (size_t m = 0; m < numMergeModules; ++m) -// { -// IdentityLayer<> identityLayer; -// identityLayer.Forward(input, identityLayer.OutputParameter()); -// -// module.Add >(identityLayer); -// } -// -// // Test the Forward function. -// module.Forward(input, output); -// REQUIRE(10 * numMergeModules == arma::accu(output)); -// -// // Test the Backward function. -// module.Backward(input, output, delta); -// REQUIRE(arma::accu(output) == arma::accu(delta)); -// } -// } - -/** - * Test the LSTM layer with a user defined rho parameter and without. - */ -// TEST_CASE("LSTMRrhoTest", "[ANNLayerTest]") -// { -// const size_t rho = 5; -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::zeros(1, 1, 5); -// RandomInitialization init(0.5, 0.5); -// -// // Create model with user defined rho parameter. -// RNN modelA( -// rho, false, NegativeLogLikelihood(), init); -// modelA.Add >(); -// modelA.Add >(1, 10); -// -// // Use LSTM layer with rho. -// modelA.Add >(10, 3, rho); -// modelA.Add >(); -// -// // Create model without user defined rho parameter. -// RNN modelB( -// rho, false, NegativeLogLikelihood(), init); -// modelB.Add >(); -// modelB.Add >(1, 10); -// -// // Use LSTM layer with rho = MAXSIZE. -// modelB.Add >(10, 3); -// modelB.Add >(); -// -// ens::StandardSGD opt(0.1, 1, 5, -100, false); -// modelA.Train(input, target, opt); -// modelB.Train(input, target, opt); -// -// CheckMatrices(modelB.Parameters(), modelA.Parameters()); -// } - -/** - * LSTM layer numerical gradient test. - */ -// TEST_CASE("GradientLSTMLayerTest", "[ANNLayerTest]") -// { -// // LSTM function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(1, 1, 5)), -// target(arma::zeros(1, 1, 5)) -// { -// const size_t rho = 5; -// -// model = new RNN(rho); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 10); -// model->Add >(10, 3, rho); -// model->Add >(); -// } -// -// ~GradientFunction() -// { -// delete model; -// } -// -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } -// -// arma::mat& Parameters() { return model->Parameters(); } -// -// RNN* model; -// arma::cube input, target; -// } function; -// -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -/** - * Test that the functions that can modify and access the parameters of the - * LSTM layer work. - */ -// TEST_CASE("LSTMLayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : inSize, outSize, rho. -// LSTM<> layer1(1, 2, 3); -// LSTM<> layer2(1, 2, 4); -// -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InSize() == 1); -// REQUIRE(layer1.OutSize() == 2); -// REQUIRE(layer1.Rho() == 3); -// -// // Now modify the parameters to match the second layer. -// layer1.Rho() = 4; -// -// // Now ensure all the results are the same. -// REQUIRE(layer1.InSize() == layer2.InSize()); -// REQUIRE(layer1.OutSize() == layer2.OutSize()); -// REQUIRE(layer1.Rho() == layer2.Rho()); -// } - -/** - * Test the FastLSTM layer with a user defined rho parameter and without. - */ -// TEST_CASE("FastLSTMRrhoTest", "[ANNLayerTest]") -// { -// const size_t rho = 5; -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::zeros(1, 1, 5); -// RandomInitialization init(0.5, 0.5); -// -// // Create model with user defined rho parameter. -// RNN modelA( -// rho, false, NegativeLogLikelihood(), init); -// modelA.Add >(); -// modelA.Add >(1, 10); -// -// // Use FastLSTM layer with rho. -// modelA.Add >(10, 3, rho); -// modelA.Add >(); -// -// // Create model without user defined rho parameter. -// RNN modelB( -// rho, false, NegativeLogLikelihood(), init); -// modelB.Add >(); -// modelB.Add >(1, 10); -// -// // Use FastLSTM layer with rho = MAXSIZE. -// modelB.Add >(10, 3); -// modelB.Add >(); -// -// ens::StandardSGD opt(0.1, 1, 5, -100, false); -// modelA.Train(input, target, opt); -// modelB.Train(input, target, opt); -// -// CheckMatrices(modelB.Parameters(), modelA.Parameters()); -// } - -/** - * FastLSTM layer numerical gradient test. - */ -// TEST_CASE("GradientFastLSTMLayerTest", "[ANNLayerTest]") -// { -// // Fast LSTM function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(1, 1, 5)), -// target(arma::zeros(1, 1, 5)) -// { -// const size_t rho = 5; -// -// model = new RNN(rho); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 10); -// model->Add >(10, 3, rho); -// model->Add >(); -// } -// -// ~GradientFunction() -// { -// delete model; -// } -// -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } -// -// arma::mat& Parameters() { return model->Parameters(); } -// -// RNN* model; -// arma::cube input, target; -// } function; -// -// // The threshold should be << 0.1 but since the Fast LSTM layer uses an -// // approximation of the sigmoid function the estimated gradient is not -// // correct. -// REQUIRE(CheckGradient(function) <= 0.2); -// } - -/** - * Test that the functions that can modify and access the parameters of the - * Fast LSTM layer work. - */ -// TEST_CASE("FastLSTMLayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : inSize, outSize, rho. -// FastLSTM<> layer1(1, 2, 3); -// FastLSTM<> layer2(1, 2, 4); -// -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InSize() == 1); -// REQUIRE(layer1.OutSize() == 2); -// REQUIRE(layer1.Rho() == 3); -// -// // Now modify the parameters to match the second layer. -// layer1.Rho() = 4; -// -// // Now ensure all the results are the same. -// REQUIRE(layer1.InSize() == layer2.InSize()); -// REQUIRE(layer1.OutSize() == layer2.OutSize()); -// REQUIRE(layer1.Rho() == layer2.Rho()); -// } - -/** - * Check whether copying and moving network with FastLSTM is working or not. - */ -// TEST_CASE("CheckCopyMoveFastLSTMTest", "[ANNLayerTest]") -// { -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::ones(1, 1, 5); -// const size_t rho = 5; -// -// RNN *model1 = -// new RNN(rho); -// model1->ResetData(input, target); -// model1->Add >(); -// model1->Add >(1, 10); -// model1->Add >(10, 3, rho); -// model1->Add >(); -// -// RNN *model2 = -// new RNN(rho); -// model2->ResetData(input, target); -// model2->Add >(); -// model2->Add >(1, 10); -// model2->Add >(10, 3, rho); -// model2->Add >(); -// -// // Check whether copy constructor is working or not. -// CheckRNNCopyFunction<>(model1, input, target, 1); -// -// // Check whether move constructor is working or not. -// CheckRNNMoveFunction<>(model2, input, target, 1); -// } - -/** - * Check whether copying and moving network with LSTM is working or not. - */ -// TEST_CASE("CheckCopyMoveLSTMTest", "[ANNLayerTest]") -// { -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::ones(1, 1, 5); -// const size_t rho = 5; -// -// RNN *model1 = -// new RNN(rho); -// model1->ResetData(input, target); -// model1->Add >(); -// model1->Add >(1, 10); -// model1->Add >(10, 3, rho); -// model1->Add >(); -// -// RNN *model2 = -// new RNN(rho); -// model2->ResetData(input, target); -// model2->Add >(); -// model2->Add >(1, 10); -// model2->Add >(10, 3, rho); -// model2->Add >(); -// -// // Check whether copy constructor is working or not. -// CheckRNNCopyFunction<>(model1, input, target, 1); -// -// // Check whether move constructor is working or not. -// CheckRNNMoveFunction<>(model2, input, target, 1); -// } - -/** - * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell - * state. Besides output, the overloaded function provides read access to cell - * state of the LSTM layer. - */ -// TEST_CASE("ReadCellStateParamLSTMLayerTest", "[ANNLayerTest]") -// { -// const size_t rho = 5, inputSize = 3, outputSize = 2; -// -// // Provide input of all ones. -// arma::cube input = arma::ones(inputSize, outputSize, rho); -// -// arma::mat inputGate, forgetGate, outputGate, hidden; -// arma::mat outLstm, cellLstm; -// -// // LSTM layer. -// LSTM<> lstm(inputSize, outputSize, rho); -// lstm.Reset(); -// lstm.ResetCell(rho); -// -// // Initialize the weights to all ones. -// lstm.Parameters().ones(); -// -// arma::mat inputWeight = arma::ones(outputSize, inputSize); -// arma::mat outputWeight = arma::ones(outputSize, outputSize); -// arma::mat bias = arma::ones(outputSize, input.n_cols); -// arma::mat cellCalc = arma::zeros(outputSize, input.n_cols); -// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); -// -// for (size_t seqNum = 0; seqNum < rho; ++seqNum) -// { -// // Wrap a matrix around our data to avoid a copy. -// arma::mat stepData(input.slice(seqNum).memptr(), -// input.n_rows, input.n_cols, false, true); -// -// // Apply Forward() on LSTM layer. -// lstm.Forward(stepData, // Input. -// outLstm, // Output. -// cellLstm, // Cell state. -// false); // Don't write into the cell state. -// -// // Compute the value of cell state and output. -// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); -// -// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); -// -// // z = tanh(W.dot(x) + W.dot(h) + b). -// hidden = arma::tanh(inputWeight * stepData + -// outputWeight * outCalc + bias); -// -// // c = f * c + i * z. -// cellCalc = forgetGate % cellCalc + inputGate % hidden; -// -// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); -// -// // h = o * tanh(c). -// outCalc = outputGate % arma::tanh(cellCalc); -// -// CheckMatrices(outLstm, outCalc, 1e-12); -// CheckMatrices(cellLstm, cellCalc, 1e-12); -// } -// } - -/** - * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell - * state. Besides output, the overloaded function provides write access to cell - * state of the LSTM layer. - */ -// TEST_CASE("WriteCellStateParamLSTMLayerTest", "[ANNLayerTest]") -// { -// const size_t rho = 5, inputSize = 3, outputSize = 2; -// -// // Provide input of all ones. -// arma::cube input = arma::ones(inputSize, outputSize, rho); -// -// arma::mat inputGate, forgetGate, outputGate, hidden; -// arma::mat outLstm, cellLstm; -// arma::mat cellCalc; -// -// // LSTM layer. -// LSTM<> lstm(inputSize, outputSize, rho); -// lstm.Reset(); -// lstm.ResetCell(rho); -// -// // Initialize the weights to all ones. -// lstm.Parameters().ones(); -// -// arma::mat inputWeight = arma::ones(outputSize, inputSize); -// arma::mat outputWeight = arma::ones(outputSize, outputSize); -// arma::mat bias = arma::ones(outputSize, input.n_cols); -// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); -// -// for (size_t seqNum = 0; seqNum < rho; ++seqNum) -// { -// // Wrap a matrix around our data to avoid a copy. -// arma::mat stepData(input.slice(seqNum).memptr(), -// input.n_rows, input.n_cols, false, true); -// -// if (cellLstm.is_empty()) -// { -// // Set the cell state to zeros. -// cellLstm = arma::zeros(outputSize, input.n_cols); -// cellCalc = arma::zeros(outputSize, input.n_cols); -// } -// else -// { -// // Set the cell state to zeros. -// cellLstm = arma::zeros(cellLstm.n_rows, cellLstm.n_cols); -// cellCalc = arma::zeros(cellCalc.n_rows, cellCalc.n_cols); -// } -// -// // Apply Forward() on the LSTM layer. -// lstm.Forward(stepData, // Input. -// outLstm, // Output. -// cellLstm, // Cell state. -// true); // Write into cell state. -// -// // Compute the value of cell state and output. -// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); -// -// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); -// -// // z = tanh(W.dot(x) + W.dot(h) + b). -// hidden = arma::tanh(inputWeight * stepData + -// outputWeight * outCalc + bias); -// -// // c = f * c + i * z. -// cellCalc = forgetGate % cellCalc + inputGate % hidden; -// -// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); -// -// // h = o * tanh(c). -// outCalc = outputGate % arma::tanh(cellCalc); -// -// CheckMatrices(outLstm, outCalc, 1e-12); -// CheckMatrices(cellLstm, cellCalc, 1e-12); -// } -// -// // Attempting to write empty matrix into cell state. -// lstm.Reset(); -// lstm.ResetCell(rho); -// arma::mat stepData(input.slice(0).memptr(), -// input.n_rows, input.n_cols, false, true); -// -// lstm.Forward(stepData, // Input. -// outLstm, // Output. -// cellLstm, // Cell state. -// true); // Write into cell state. -// -// for (size_t seqNum = 1; seqNum < rho; ++seqNum) -// { -// arma::mat empty; -// // Should throw error. -// REQUIRE_THROWS_AS(lstm.Forward(stepData, // Input. -// outLstm, // Output. -// empty, // Cell state. -// true), // Write into cell state. -// std::runtime_error); -// } -// } - -/** - * Test that the functions that can modify and access the parameters of the - * GRU layer work. - */ -// TEST_CASE("GRULayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : inSize, outSize, rho. -// GRU<> layer1(1, 2, 3); -// GRU<> layer2(1, 2, 4); -// -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InSize() == 1); -// REQUIRE(layer1.OutSize() == 2); -// REQUIRE(layer1.Rho() == 3); -// -// // Now modify the parameters to match the second layer. -// layer1.Rho() = 4; -// -// // Now ensure all the results are the same. -// REQUIRE(layer1.InSize() == layer2.InSize()); -// REQUIRE(layer1.OutSize() == layer2.OutSize()); -// REQUIRE(layer1.Rho() == layer2.Rho()); -// } - -/** - * Check if the gradients computed by GRU cell are close enough to the - * approximation of the gradients. - */ -// TEST_CASE("GradientGRULayerTest", "[ANNLayerTest]") -// { -// // GRU function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(1, 1, 5)), -// target(arma::zeros(1, 1, 5)) -// { -// const size_t rho = 5; -// -// model = new RNN(rho); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 10); -// model->Add >(10, 3, rho); -// model->Add >(); -// } -// -// ~GradientFunction() -// { -// delete model; -// } -// -// double Gradient(arma::mat& gradient) const -// { -// arma::mat output; -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } -// -// arma::mat& Parameters() { return model->Parameters(); } -// -// RNN* model; -// arma::cube input, target; -// } function; -// -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -/** - * GRU layer manual forward test. - */ -// TEST_CASE("ForwardGRULayerTest", "[ANNLayerTest]") -// { -// // This will make it easier to clean memory later. -// GRU<>* gruAlloc = new GRU<>(3, 3, 5); -// GRU<>& gru = *gruAlloc; -// -// // Initialize the weights to all ones. -// NetworkInitialization -// networkInit(ConstInitialization(1)); -// networkInit.Initialize(gru.Model(), gru.Parameters()); -// -// // Provide input of all ones. -// arma::mat input = arma::ones(3, 1); -// arma::mat output; -// -// gru.Forward(input, output); -// -// // Compute the z_t gate output. -// arma::mat expectedOutput = arma::ones(3, 1); -// expectedOutput *= -4; -// expectedOutput = arma::exp(expectedOutput); -// expectedOutput = arma::ones(3, 1) / (arma::ones(3, 1) + expectedOutput); -// expectedOutput = (arma::ones(3, 1) - expectedOutput) % expectedOutput; -// -// // For the first input the output should be equal to the output of -// // gate z_t as the previous output fed to the cell is all zeros. -// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); -// -// expectedOutput = output; -// -// gru.Forward(input, output); -// -// double s = arma::as_scalar(arma::sum(expectedOutput)); -// -// // Compute the value of z_t gate for the second input. -// arma::mat z_t = arma::ones(3, 1); -// z_t *= -(s + 4); -// z_t = arma::exp(z_t); -// z_t = arma::ones(3, 1) / (arma::ones(3, 1) + z_t); -// -// // Compute the value of o_t gate for the second input. -// arma::mat o_t = arma::ones(3, 1); -// o_t *= -(arma::as_scalar(arma::sum(expectedOutput % z_t)) + 4); -// o_t = arma::exp(o_t); -// o_t = arma::ones(3, 1) / (arma::ones(3, 1) + o_t); -// -// // Expected output for the second input. -// expectedOutput = z_t % expectedOutput + (arma::ones(3, 1) - z_t) % o_t; -// -// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); -// -// LayerTypes<> layer(gruAlloc); -// boost::apply_visitor(DeleteVisitor(), layer); -// } - -/** - * Simple lookup module test. - * -TEST_CASE("SimpleLookupLayerTest", "[ANNLayerTest]") -{ - const size_t vocabSize = 10; - const size_t embeddingSize = 2; - const size_t seqLength = 3; - const size_t batchSize = 4; - - arma::mat output, input, gy, g, gradient; - - Lookup module(vocabSize, embeddingSize); - module.Parameters().randu(); - - // Test the Forward function. - input = arma::zeros(seqLength, batchSize); - for (size_t i = 0; i < input.n_elem; ++i) - { - int token = RandInt(1, vocabSize); - input(i) = token; - } - - module.Forward(input, output); - for (size_t i = 0; i < batchSize; ++i) - { - // The Lookup module uses index - 1 for the cols. - const double outputSum = arma::accu(module.Parameters().cols( - arma::conv_to::from(input.col(i)) - 1)); - - REQUIRE(std::fabs(outputSum - arma::accu(output.col(i))) <= 1e-5); - } - - // Test the Gradient function. - arma::mat error = 0.01 * arma::randu(embeddingSize * seqLength, batchSize); - module.Gradient(input, error, gradient); - - REQUIRE(std::fabs(arma::accu(error) - arma::accu(gradient)) <= 1e-07); -} -*/ - -/** - * Lookup layer numerical gradient test. - * -TEST_CASE("GradientLookupLayerTest", "[ANNLayerTest]") -{ - // Lookup function gradient instantiation. - struct GradientFunction - { - GradientFunction() - { - input.set_size(seqLength, batchSize); - for (size_t i = 0; i < input.n_elem; ++i) - { - input(i) = RandInt(1, vocabSize); - } - target = arma::zeros(vocabSize, batchSize); - for (size_t i = 0; i < batchSize; ++i) - { - const size_t targetWord = RandInt(1, vocabSize); - target(targetWord, i) = 1; - } - - model = new FFN, GlorotInitialization>(BCELoss<>(1e-10, false)); - model->ResetData(input, target); - model->Add(vocabSize, embeddingSize); - model->Add(embeddingSize * seqLength, vocabSize); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, batchSize); - model->Gradient(model->Parameters(), 0, gradient, batchSize); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN, GlorotInitialization>* model; - arma::mat input, target; - - const size_t seqLength = 10; - const size_t embeddingSize = 8; - const size_t vocabSize = 20; - const size_t batchSize = 4; - } function; - - REQUIRE(CheckGradient(function) <= 1e-6); -} -*/ - -/** - * Test that the functions that can access the parameters of the - * Lookup layer work. - * -TEST_CASE("LookupLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : vocabSize, embedingSize. - Lookup layer(100, 8); - - // Make sure we can get the parameters successfully. - REQUIRE(layer.VocabSize() == 100); - REQUIRE(layer.EmbeddingSize() == 8); -} -*/ - -/** - * Simple test for the NearestInterpolation layer - * -TEST_CASE("SimpleNearestInterpolationLayerTest", "[ANNLayerTest]") -{ - // Tested output against torch.nn.Upsample(mode="nearest"). - arma::mat input, output, unzoomedOutput, expectedOutput; - size_t inRowSize = 2; - size_t inColSize = 2; - size_t outRowSize = 5; - size_t outColSize = 7; - size_t depth = 1; - input.zeros(inRowSize * inColSize * depth, 1); - input[0] = 1.0; - input[1] = 3.0; - input[2] = 2.0; - input[3] = 4.0; - NearestInterpolation<> layer(inRowSize, inColSize, outRowSize, - outColSize, depth); - - expectedOutput << 1.0000 << 1.0000 << 1.0000 << 1.0000 << 2.0000 - << 2.0000 << 2.0000 << arma::endr - << 1.0000 << 1.0000 << 1.0000 << 1.0000 << 2.0000 - << 2.0000 << 2.0000 << arma::endr - << 1.0000 << 1.0000 << 1.0000 << 1.0000 << 2.0000 - << 2.0000 << 2.0000 << arma::endr - << 3.0000 << 3.0000 << 3.0000 << 3.0000 << 4.0000 - << 4.0000 << 4.0000 << arma::endr - << 3.0000 << 3.0000 << 3.0000 << 3.0000 << 4.0000 - << 4.0000 << 4.0000 << arma::endr; - expectedOutput.reshape(35, 1); - - layer.Forward(input, output); - CheckMatrices(output - expectedOutput, - arma::zeros(output.n_rows), 1e-4); - - expectedOutput.clear(); - expectedOutput << 12.0000 << 18.0000 << arma::endr - << 24.0000 << 24.0000 << arma::endr; - expectedOutput.reshape(4, 1); - layer.Backward(output, output, unzoomedOutput); - CheckMatrices(unzoomedOutput - expectedOutput, - arma::zeros(input.n_rows), 1e-4); - - arma::mat input1, output1, unzoomedOutput1, expectedOutput1; - inRowSize = 2; - inColSize = 3; - outRowSize = 17; - outColSize = 23; - input1 << 1 << 2 << 3 << arma::endr - << 4 << 5 << 6 << arma::endr; - input1.reshape(6, 1); - NearestInterpolation<> layer1(inRowSize, inColSize, outRowSize, - outColSize, depth); - - layer1.Forward(input1, output1); - layer1.Backward(output1, output1, unzoomedOutput1); - - REQUIRE(arma::accu(output1) - 1317.00 == Approx(0.0).margin(1e-05)); - REQUIRE(arma::accu(unzoomedOutput1) - 1317.00 == - Approx(0.0).margin(1e-05)); -} -*/ - -/* - * Simple test for the BilinearInterpolation layer - * -TEST_CASE("SimpleBilinearInterpolationLayerTest", "[ANNLayerTest]") -{ - // Tested output against tensorflow.image.resize_bilinear() - arma::mat input, output, unzoomedOutput, expectedOutput; - size_t inRowSize = 2; - size_t inColSize = 2; - size_t outRowSize = 5; - size_t outColSize = 5; - size_t depth = 1; - input.zeros(inRowSize * inColSize * depth, 1); - input[0] = 1.0; - input[1] = input[2] = 2.0; - input[3] = 3.0; - BilinearInterpolation layer(inRowSize, inColSize, outRowSize, outColSize, - depth); - expectedOutput = arma::mat("1.0000 1.4000 1.8000 2.0000 2.0000 \ - 1.4000 1.8000 2.2000 2.4000 2.4000 \ - 1.8000 2.2000 2.6000 2.8000 2.8000 \ - 2.0000 2.4000 2.8000 3.0000 3.0000 \ - 2.0000 2.4000 2.8000 3.0000 3.0000"); - expectedOutput.reshape(25, 1); - layer.Forward(input, output); - CheckMatrices(output - expectedOutput, arma::zeros(output.n_rows), 1e-12); - - expectedOutput = arma::mat("1.0000 1.9000 1.9000 2.8000"); - expectedOutput.reshape(4, 1); - layer.Backward(output, output, unzoomedOutput); - CheckMatrices(unzoomedOutput - expectedOutput, - arma::zeros(input.n_rows), 1e-12); -} -*/ - -/** - * Test that the functions that can modify and access the parameters of the - * Bilinear Interpolation layer work. - * -TEST_CASE("BilinearInterpolationLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : inRowSize, inColSize, outRowSize, outColSize, depth. - BilinearInterpolation layer1(1, 2, 3, 4, 5); - BilinearInterpolation layer2(2, 3, 4, 5, 6); - - // Make sure we can get the parameters successfully. - REQUIRE(layer1.InRowSize() == 1); - REQUIRE(layer1.InColSize() == 2); - REQUIRE(layer1.OutRowSize() == 3); - REQUIRE(layer1.OutColSize() == 4); - REQUIRE(layer1.InDepth() == 5); - - // Now modify the parameters to match the second layer. - layer1.InRowSize() = 2; - layer1.InColSize() = 3; - layer1.OutRowSize() = 4; - layer1.OutColSize() = 5; - layer1.InDepth() = 6; - - // Now ensure all results are the same. - REQUIRE(layer1.InRowSize() == layer2.InRowSize()); - REQUIRE(layer1.InColSize() == layer2.InColSize()); - REQUIRE(layer1.OutRowSize() == layer2.OutRowSize()); - REQUIRE(layer1.OutColSize() == layer2.OutColSize()); - REQUIRE(layer1.InDepth() == layer2.InDepth()); -} -*/ - -/* - * Simple test for the BicubicInterpolation layer. - * -TEST_CASE("SimpleBicubicInterpolationLayerTest", "[ANNLayerTest]") -{ - // Tested output against torch.nn.Upsample(mode="bicubic"). - // Test case with square input with rectangular output. - arma::mat input, output, unzoomedOutput, expectedOutput; - size_t inRowSize = 2; - size_t inColSize = 2; - size_t outRowSize = 5; - size_t outColSize = 7; - size_t depth = 1; - input.zeros(inRowSize * inColSize * depth, 1); - - input << 10 << 20 << arma::endr - << 30 << 40 << arma::endr; - input.reshape(4, 1); - BicubicInterpolation<> layer(inRowSize, inColSize, outRowSize, - outColSize, depth); - - expectedOutput << 6.68803935860 << 7.33308309038 << 9.69733236152 - << 12.79500000000 << 15.89266763848 << 18.25691690962 - << 18.90196064140 << arma::endr - << 10.53303935860 << 11.17808309038 << 13.54233236152 - << 16.64000000000 << 19.73766763848 << 22.10191690962 - << 22.74696064140 << arma::endr - << 18.89303935860 << 19.53808309038 << 21.90233236152 - << 25.00000000000 << 28.09766763848 << 30.46191690962 - << 31.10696064140 << arma::endr - << 27.25303935860 << 27.89808309038 << 30.26233236152 - << 33.36000000000 << 36.45766763848 << 38.82191690962 - << 39.46696064140 << arma::endr - << 31.09803935860 << 31.74308309038 << 34.10733236152 - << 37.20500000000 << 40.30266763848 << 42.66691690962 - << 43.31196064140 << arma::endr; - expectedOutput.reshape(35, 1); - layer.Forward(input, output); - - CheckMatrices(output, expectedOutput, 1e-6); - - expectedOutput.clear(); - expectedOutput << 103.79040654914 << 180.51345595086 << arma::endr - << 256.98654404914 << 333.70959345086 << arma::endr; - expectedOutput.reshape(4, 1); - - layer.Backward(output, output, unzoomedOutput); - - CheckMatrices(unzoomedOutput, expectedOutput, 1e-6); - - // Tested output against torch.nn.Upsample(mode="bicubic"). - // Test case with rectangular input with rectangular output. - arma::mat input1, output1, unzoomedOutput1, expectedOutput1, expectedUnzoomed; - - inRowSize = 2; - inColSize = 3; - outRowSize = 5; - outColSize = 7; - depth = 1; - input1.zeros(inRowSize * inColSize * depth, 1); - - input1 << 10 << 20 << 30 << arma::endr - << 40 << 50 << 60 << arma::endr; - input1.reshape(6, 1); - - BicubicInterpolation<> layer1(inRowSize, inColSize, outRowSize, - outColSize, depth); - - expectedOutput1 << 5.59920553936 << 7.77121720117 << 11.44468658892 - << 16.69250000000 << 21.94031341108 << 25.61378279883 - << 27.78579446064 << arma::endr - << 11.36670553936 << 13.53871720117 << 17.21218658892 - << 22.46000000000 << 27.70781341108 << 31.38128279883 - << 33.55329446064 << arma::endr - << 23.90670553936 << 26.07871720117 << 29.75218658892 - << 35.00000000000 << 40.24781341108 << 43.92128279883 - << 46.09329446064 << arma::endr - << 36.44670553936 << 38.61871720117 << 42.29218658892 - << 47.54000000000 << 52.78781341108 << 56.46128279883 - << 58.63329446064 << arma::endr - << 42.21420553936 << 44.38621720117 << 48.05968658892 - << 53.30750000000 << 58.55531341108 << 62.22878279883 - << 64.40079446064 << arma::endr; - expectedOutput1.reshape(35, 1); - layer1.Forward(input1, output1); - - CheckMatrices(output1, expectedOutput1, 1e-6); - - expectedUnzoomed << 67.65674505130 << 132.29729646501 - << 182.75175223368 << arma::endr - << 218.01355388877 << 291.17209129009 - << 333.10856107115 << arma::endr; - expectedUnzoomed.reshape(6, 1); - - layer1.Backward(output1, output1, unzoomedOutput1); - CheckMatrices(unzoomedOutput1, expectedUnzoomed, 1e-6); -} -*/ - -/** - * VirtualBatchNorm layer numerical gradient test. - * -TEST_CASE("GradientVirtualBatchNormTest", "[ANNLayerTest]") -{ - // Add function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - input(arma::randn(5, 256)), - target(arma::zeros(1, 256)) - { - arma::mat referenceBatch = arma::mat(input.memptr(), input.n_rows, 4); - - model = new FFN(); - model->ResetData(input, target); - model->Add(); - model->Add(5, 5); - model->Add(referenceBatch, 5); - model->Add(5, 2); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 16, false); - model->Gradient(model->Parameters(), 0, gradient, 16); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - REQUIRE(CheckGradient(function) <= 1e-4); -} -*/ - -/** - * Test that the functions that can modify and access the parameters of the - * Virtual Batch Norm layer work. - * -TEST_CASE("VirtualBatchNormLayerParametersTest", "[ANNLayerTest]") -{ - arma::mat input = arma::randn(5, 16); - arma::mat referenceBatch = arma::mat(input.memptr(), input.n_rows, 4); - - // Parameter order : referenceBatch, size, eps. - VirtualBatchNorm layer(referenceBatch, 5, 1e-3); - - // Make sure we can get the parameters successfully. - REQUIRE(layer.InSize() == 5); - REQUIRE(layer.Epsilon() == 1e-3); -} -*/ - -// /** -// * MiniBatchDiscrimination layer numerical gradient test. -// */ -// TEST_CASE("MiniBatchDiscriminationTest", "[ANNLayerTest]") -// { -// // Add function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randn(5, 4)), -// target(arma::zeros(1, 4)) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(5, 5); -// model->Add >(5, 10, 16); -// model->Add >(10, 2); -// model->Add >(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// return model->EvaluateWithGradient(model->Parameters(), 0, gradient, 4); -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// arma::mat input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -/** - * Simple Transposed Convolution layer test. - * -TEST_CASE("SimpleTransposedConvolutionLayerTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta; - - TransposedConvolution module1(1, 1, 3, 3, 1, 1, 0, 0, 4, 4, 6, 6); - // Test the forward function. - input = arma::linspace(0, 15, 16); - module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); - module1.Parameters()(0) = 1.0; - module1.Parameters()(8) = 2.0; - module1.Reset(); - module1.Forward(input, output); - // Value calculated using tensorflow.nn.conv2d_transpose() - REQUIRE(arma::accu(output) == 360.0); - - // Test the backward function. - module1.Backward(input, output, delta); - // Value calculated using tensorflow.nn.conv2d() - REQUIRE(arma::accu(delta) == 720.0); - - TransposedConvolution module2(1, 1, 4, 4, 1, 1, 1, 1, 5, 5, 6, 6); - // Test the forward function. - input = arma::linspace(0, 24, 25); - module2.Parameters() = arma::mat(16 + 1, 1, arma::fill::zeros); - module2.Parameters()(0) = 1.0; - module2.Parameters()(3) = 1.0; - module2.Parameters()(6) = 1.0; - module2.Parameters()(9) = 1.0; - module2.Parameters()(12) = 1.0; - module2.Parameters()(15) = 2.0; - module2.Reset(); - module2.Forward(input, output); - // Value calculated using torch.nn.functional.conv_transpose2d() - REQUIRE(arma::accu(output) == 1512.0); - - // Test the backward function. - module2.Backward(input, output, delta); - // Value calculated using torch.nn.functional.conv2d() - REQUIRE(arma::accu(delta) == 6504.0); - - TransposedConvolution module3(1, 1, 3, 3, 1, 1, 1, 1, 5, 5, 5, 5); - // Test the forward function. - input = arma::linspace(0, 24, 25); - module3.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); - module3.Parameters()(1) = 2.0; - module3.Parameters()(2) = 4.0; - module3.Parameters()(3) = 3.0; - module3.Parameters()(8) = 1.0; - module3.Reset(); - module3.Forward(input, output); - // Value calculated using torch.nn.functional.conv_transpose2d() - REQUIRE(arma::accu(output) == 2370.0); - - // Test the backward function. - module3.Backward(input, output, delta); - // Value calculated using torch.nn.functional.conv2d() - REQUIRE(arma::accu(delta) == 19154.0); - - TransposedConvolution module4(1, 1, 3, 3, 1, 1, 0, 0, 5, 5, 7, 7); - // Test the forward function. - input = arma::linspace(0, 24, 25); - module4.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); - module4.Parameters()(2) = 2.0; - module4.Parameters()(4) = 4.0; - module4.Parameters()(6) = 6.0; - module4.Parameters()(8) = 8.0; - module4.Reset(); - module4.Forward(input, output); - // Value calculated using torch.nn.functional.conv_transpose2d() - REQUIRE(arma::accu(output) == 6000.0); - - // Test the backward function. - module4.Backward(input, output, delta); - // Value calculated using torch.nn.functional.conv2d() - REQUIRE(arma::accu(delta) == 86208.0); - - TransposedConvolution module5(1, 1, 3, 3, 2, 2, 0, 0, 2, 2, 5, 5); - // Test the forward function. - input = arma::linspace(0, 3, 4); - module5.Parameters() = arma::mat(25 + 1, 1, arma::fill::zeros); - module5.Parameters()(2) = 8.0; - module5.Parameters()(4) = 6.0; - module5.Parameters()(6) = 4.0; - module5.Parameters()(8) = 2.0; - module5.Reset(); - module5.Forward(input, output); - // Value calculated using torch.nn.functional.conv_transpose2d() - REQUIRE(arma::accu(output) == 120.0); - - // Test the backward function. - module5.Backward(input, output, delta); - // Value calculated using torch.nn.functional.conv2d() - REQUIRE(arma::accu(delta) == 960.0); - - TransposedConvolution module6(1, 1, 3, 3, 2, 2, 1, 1, 3, 3, 5, 5); - // Test the forward function. - input = arma::linspace(0, 8, 9); - module6.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); - module6.Parameters()(0) = 8.0; - module6.Parameters()(3) = 6.0; - module6.Parameters()(6) = 2.0; - module6.Parameters()(8) = 4.0; - module6.Reset(); - module6.Forward(input, output); - // Value calculated using torch.nn.functional.conv_transpose2d() - REQUIRE(arma::accu(output) == 410.0); - - // Test the backward function. - module6.Backward(input, output, delta); - // Value calculated using torch.nn.functional.conv2d() - REQUIRE(arma::accu(delta) == 4444.0); - - TransposedConvolution module7(1, 1, 3, 3, 2, 2, 1, 1, 3, 3, 6, 6); - // Test the forward function. - input = arma::linspace(0, 8, 9); - module7.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); - module7.Parameters()(0) = 8.0; - module7.Parameters()(2) = 6.0; - module7.Parameters()(4) = 2.0; - module7.Parameters()(8) = 4.0; - module7.Reset(); - module7.Forward(input, output); - // Value calculated using torch.nn.functional.conv_transpose2d() - REQUIRE(arma::accu(output) == 606.0); - - module7.Backward(input, output, delta); - // Value calculated using torch.nn.functional.conv2d() - REQUIRE(arma::accu(delta) == 7732.0); -} -*/ - -/** - * Transposed Convolution layer numerical gradient test. - * -TEST_CASE("GradientTransposedConvolutionLayerTest", "[ANNLayerTest]") -{ - // Add function gradient instantiation. - // To make this test robust, check it five times. - bool pass = false; - for (size_t trial = 0; trial < 5; trial++) - { - struct GradientFunction - { - GradientFunction() : - input(arma::linspace(0, 35, 36)), - target(arma::mat("0")) - { - model = new FFN(); - model->ResetData(input, target); - model->Add(1, 1, 3, 3, 2, 2, 1, 1, 6, 6, 12, 12); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - if (CheckGradient(function) < 1e-3) - { - pass = true; - break; - } - } - REQUIRE(pass == true); -} -*/ - -/** - * Simple MultiplyMerge module test. - * -TEST_CASE("SimpleMultiplyMergeLayerTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta; - input = arma::ones(10, 1); - - for (size_t i = 0; i < 5; ++i) - { - MultiplyMerge module(false, false); - const size_t numMergeModules = RandInt(2, 10); - for (size_t m = 0; m < numMergeModules; ++m) - { - IdentityLayer* identityLayer = new IdentityLayer(); - identityLayer->Forward(input, identityLayer->OutputParameter()); - - module.Add(identityLayer); - } - - // Test the Forward function. - module.Forward(input, output); - REQUIRE(10 == arma::accu(output)); - - // Test the Backward function. - module.Backward(input, output, delta); - REQUIRE(arma::accu(output) == arma::accu(delta)); - } -} -*/ - -/** - * Check whether copying and moving network with MultiplyMerge is working or - * not. - */ -// TEST_CASE("CheckCopyMoveMultiplyMergeTest", "[ANNLayerTest]") -// { -// arma::mat input(10, 1); -// input.randu(); -// -// arma::mat output1; -// arma::mat output2; -// arma::mat output3; -// arma::mat output4; -// -// const size_t numMergeModules = RandInt(2, 10); -// -// MultiplyMerge<> *module1 = new MultiplyMerge<>(true, false); -// for (size_t m = 0; m < numMergeModules; ++m) -// { -// IdentityLayer<> identityLayer; -// identityLayer.Forward(input, identityLayer.OutputParameter()); -// -// module1->Add >(identityLayer); -// } -// -// module1->Forward(input, output1); -// -// MultiplyMerge<> module2 = *module1; -// delete module1; -// -// module2.Forward(input, output2); -// CheckMatrices(output1, output2); -// -// MultiplyMerge<> *module3 = new MultiplyMerge<>(true, false); -// for (size_t m = 0; m < numMergeModules; ++m) -// { -// IdentityLayer<> identityLayer; -// identityLayer.Forward(input, identityLayer.OutputParameter()); -// -// module3->Add >(identityLayer); -// } -// module3->Forward(input, output3); -// -// MultiplyMerge<> module4(std::move(*module3)); -// delete module3; -// -// module4.Forward(input, output4); -// CheckMatrices(output3, output4); -// } - -// /** -// * Simple Atrous Convolution layer test. -// */ -// TEST_CASE("SimpleAtrousConvolutionLayerTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta; - -// AtrousConvolution<> module1(1, 1, 3, 3, 1, 1, 0, 0, 7, 7, 2, 2); -// // Test the Forward function. -// input = arma::linspace(0, 48, 49); -// module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); -// module1.Parameters()(0) = 1.0; -// module1.Parameters()(8) = 2.0; -// module1.Reset(); -// module1.Forward(input, output); -// // Value calculated using tensorflow.nn.atrous_conv2d() -// REQUIRE(arma::accu(output) == 792.0); - -// // Test the Backward function. -// module1.Backward(input, output, delta); -// REQUIRE(arma::accu(delta) == 2376); - -// AtrousConvolution<> module2(1, 1, 3, 3, 2, 2, 0, 0, 7, 7, 2, 2); -// // Test the forward function. -// input = arma::linspace(0, 48, 49); -// module2.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); -// module2.Parameters()(0) = 1.0; -// module2.Parameters()(3) = 1.0; -// module2.Parameters()(6) = 1.0; -// module2.Reset(); -// module2.Forward(input, output); -// // Value calculated using tensorflow.nn.conv2d() -// REQUIRE(arma::accu(output) == 264.0); - -// // Test the backward function. -// module2.Backward(input, output, delta); -// REQUIRE(arma::accu(delta) == 792.0); -// } - -// /** -// * Atrous Convolution layer numerical gradient test. -// */ -// TEST_CASE("GradientAtrousConvolutionLayerTest", "[ANNLayerTest]") -// { -// // Add function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::linspace(0, 35, 36)), -// target(arma::mat("0")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 1, 3, 3, 1, 1, 0, 0, 6, 6, 2, 2); -// model->Add >(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// arma::mat input, target; -// } function; - -// // TODO: this tolerance seems far higher than necessary. The implementation -// // should be checked. -// REQUIRE(CheckGradient(function) <= 0.2); -// } - -// /** -// * Test the functions to access and modify the parameters of the -// * AtrousConvolution layer. -// */ -// TEST_CASE("AtrousConvolutionLayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order for the constructor: inSize, outSize, kW, kH, dW, dH, padW, -// // padH, inputWidth, inputHeight, dilationW, dilationH, paddingType ("none"). -// AtrousConvolution<> layer1(1, 2, 3, 4, 5, 6, std::make_tuple(7, 8), -// std::make_tuple(9, 10), 11, 12, 13, 14); -// AtrousConvolution<> layer2(2, 3, 4, 5, 6, 7, std::make_tuple(8, 9), -// std::make_tuple(10, 11), 12, 13, 14, 15); - -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InputWidth() == 11); -// REQUIRE(layer1.InputHeight() == 12); -// REQUIRE(layer1.KernelWidth() == 3); -// REQUIRE(layer1.KernelHeight() == 4); -// REQUIRE(layer1.StrideWidth() == 5); -// REQUIRE(layer1.StrideHeight() == 6); -// REQUIRE(layer1.Padding().PadHTop() == 9); -// REQUIRE(layer1.Padding().PadHBottom() == 10); -// REQUIRE(layer1.Padding().PadWLeft() == 7); -// REQUIRE(layer1.Padding().PadWRight() == 8); -// REQUIRE(layer1.DilationWidth() == 13); -// REQUIRE(layer1.DilationHeight() == 14); - -// // Now modify the parameters to match the second layer. -// layer1.InputWidth() = 12; -// layer1.InputHeight() = 13; -// layer1.KernelWidth() = 4; -// layer1.KernelHeight() = 5; -// layer1.StrideWidth() = 6; -// layer1.StrideHeight() = 7; -// layer1.Padding().PadHTop() = 10; -// layer1.Padding().PadHBottom() = 11; -// layer1.Padding().PadWLeft() = 8; -// layer1.Padding().PadWRight() = 9; -// layer1.DilationWidth() = 14; -// layer1.DilationHeight() = 15; - -// // Now ensure all results are the same. -// REQUIRE(layer1.InputWidth() == layer2.InputWidth()); -// REQUIRE(layer1.InputHeight() == layer2.InputHeight()); -// REQUIRE(layer1.KernelWidth() == layer2.KernelWidth()); -// REQUIRE(layer1.KernelHeight() == layer2.KernelHeight()); -// REQUIRE(layer1.StrideWidth() == layer2.StrideWidth()); -// REQUIRE(layer1.StrideHeight() == layer2.StrideHeight()); -// REQUIRE(layer1.Padding().PadHTop() == layer2.Padding().PadHTop()); -// REQUIRE(layer1.Padding().PadHBottom() == -// layer2.Padding().PadHBottom()); -// REQUIRE(layer1.Padding().PadWLeft() == -// layer2.Padding().PadWLeft()); -// REQUIRE(layer1.Padding().PadWRight() == -// layer2.Padding().PadWRight()); -// REQUIRE(layer1.DilationWidth() == layer2.DilationWidth()); -// REQUIRE(layer1.DilationHeight() == layer2.DilationHeight()); -// } - -// /** -// * Test that the padding options are working correctly in Atrous Convolution -// * layer. -// */ -// TEST_CASE("AtrousConvolutionLayerPaddingTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta; - -// // Check valid padding option. -// AtrousConvolution<> module1(1, 1, 3, 3, 1, 1, -// std::tuple(1, 1), std::tuple(1, 1), 7, 7, -// 2, 2, "valid"); - -// // Test the Forward function. -// input = arma::linspace(0, 48, 49); -// module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); -// module1.Reset(); -// module1.Forward(input, output); - -// REQUIRE(arma::accu(output) == 0); -// REQUIRE(output.n_rows == 9); -// REQUIRE(output.n_cols == 1); - -// // Test the Backward function. -// module1.Backward(input, output, delta); - -// // Check same padding option. -// AtrousConvolution<> module2(1, 1, 3, 3, 1, 1, -// std::tuple(0, 0), std::tuple(0, 0), 7, 7, -// 2, 2, "same"); - -// // Test the forward function. -// input = arma::linspace(0, 48, 49); -// module2.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); -// module2.Reset(); -// module2.Forward(input, output); - -// REQUIRE(arma::accu(output) == 0); -// REQUIRE(output.n_rows == 49); -// REQUIRE(output.n_cols == 1); - -// // Test the backward function. -// module2.Backward(input, output, delta); -// } - -/** - * Tests the GroupNorm layer. - */ -// TEST_CASE("GroupNormTest", "[ANNLayerTest]") -// { -// arma::mat input, output, backwardOutput; -// input = { -// { 2, 0, 1 }, -// { 3, 1, 2 }, -// { 5, 1, 3 }, -// { 7, 2, 4 }, -// { 11, 3, 5 }, -// { 13, 5, 6 }, -// { 17, 8, 7 }, -// { 19, 13, 8 } -// }; -// -// GroupNorm<> model(2, 4); -// model.Reset(); -// -// model.Forward(input, output); -// arma::mat result; -// result = { -// { -1.1717001972, -1.4142135482, -1.3416407811 }, -// { -0.6509445540, 0.0000000000 , -0.4472135937 }, -// { 0.3905667324 , 0.0000000000 , 0.4472135937 }, -// { 1.4320780188 , 1.4142135482 , 1.341640781 }, -// { -1.2649110634, -1.1283296293, -1.3416407811 }, -// { -0.6324555317, -0.5973509802, -0.4472135937 }, -// { 0.6324555317 , 0.1991169934 , 0.4472135937 }, -// { 1.2649110634 , 1.5265636161 , 1.3416407811 } -// }; -// -// CheckMatrices(output, result, 1e-5); -// } - -/** - * GroupNorm layer numerical gradient test. - */ -// TEST_CASE("GradientGroupNormTest", "[ANNLayerTest]") -// { -// // Add function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randn(10, 256)), -// target(arma::zeros(1, 256)) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(10, 10); -// model->Add >(1, 10); -// model->Add >(10, 2); -// model->Add >(); -// } -// -// ~GradientFunction() -// { -// delete model; -// } -// -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 256, false); -// model->Gradient(model->Parameters(), 0, gradient, 256); -// return error; -// } -// -// arma::mat& Parameters() { return model->Parameters(); } -// -// FFN* model; -// arma::mat input, target; -// } function; -// -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -/** - * Tests the LayerNorm layer. - * -TEST_CASE("LayerNormTest", "[ANNLayerTest]") -{ - arma::mat input, output; - input = { { 5.1, 3.5 }, - { 4.9, 3.0 }, - { 4.7, 3.2 } }; - - LayerNorm model(input.n_rows); - model.Reset(); - - model.Forward(input, output); - arma::mat result; - result = { { 1.2247, 1.2978 }, - { 0, -1.1355 }, - { -1.2247, -0.1622 } }; - - CheckMatrices(output, result, 1e-1); - result.clear(); - - output = model.Mean(); - result = { 4.9000, 3.2333 }; - - CheckMatrices(output, result, 1e-1); - result.clear(); - - output = model.Variance(); - result = { 0.0267, 0.0422 }; - - CheckMatrices(output, result, 1e-1); -} -*/ - -/** - * LayerNorm layer numerical gradient test. - * -TEST_CASE("GradientLayerNormTest", "[ANNLayerTest]") -{ - // Add function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - input(arma::randn(10, 256)), - target(arma::zeros(1, 256)) - { - model = new FFN(); - model->ResetData(input, target); - model->Add(); - model->Add(10, 10); - model->Add(10); - model->Add(10, 2); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 16, false); - model->Gradient(model->Parameters(), 0, gradient, 16); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - REQUIRE(CheckGradient(function) <= 1e-4); -} -*/ - -/** - * Test that the functions that can access the parameters of the - * Layer Norm layer work. - * -TEST_CASE("LayerNormLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : size, eps. - LayerNorm layer(5, 1e-3); - - // Make sure we can get the parameters successfully. - REQUIRE(layer.InSize() == 5); - REQUIRE(layer.Epsilon() == 1e-3); -} -*/ - -// /** -// * Test if the AddMerge layer is able to forward the -// * Forward/Backward/Gradient calls. -// */ -// TEST_CASE("AddMergeRunTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta, error; - -// AddMerge<> module(true, true); - -// Linear<>* linear = new Linear<>(10, 10); -// module.Add(linear); - -// linear->Parameters().randu(); -// linear->Reset(); - -// input = arma::zeros(10, 1); -// module.Forward(input, output); - -// double parameterSum = arma::accu(linear->Parameters().submat( -// 100, 0, linear->Parameters().n_elem - 1, 0)); - -// // Test the Backward function. -// module.Backward(input, input, delta); - -// // Clean up before we break, -// delete linear; - -// REQUIRE(parameterSum == Approx(arma::accu(output)).epsilon(1e-5)); -// REQUIRE(arma::accu(delta) == 0); -// } - -/** - * Test if the MultiplyMerge layer is able to forward the - * Forward/Backward/Gradient calls. - * -TEST_CASE("MultiplyMergeRunTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta, error; - - MultiplyMerge module(true, true); - - Linear* linear = new Linear(10, 10); - module.Add(linear); - - linear->Parameters().randu(); - linear->Reset(); - - input = arma::zeros(10, 1); - module.Forward(input, output); - - double parameterSum = arma::accu(linear->Parameters().submat( - 100, 0, linear->Parameters().n_elem - 1, 0)); - - // Test the Backward function. - module.Backward(input, input, delta); - - // Clean up before we break, - delete linear; - - REQUIRE(parameterSum == Approx(arma::accu(output)).epsilon(1e-5)); - REQUIRE(arma::accu(delta) == 0); -} -*/ - -/** - * Simple subview module test. - * -TEST_CASE("SimpleSubviewLayerTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta, outputMat; - Subview moduleRow(1, 10, 19); - - // Test the Forward function for a vector. - input = arma::ones(20, 1); - moduleRow.Forward(input, output); - REQUIRE(output.n_rows == 10); - - Subview moduleMat(4, 3, 6, 0, 2); - - // Test the Forward function for a matrix. - input = arma::ones(20, 8); - moduleMat.Forward(input, outputMat); - REQUIRE(outputMat.n_rows == 12); - REQUIRE(outputMat.n_cols == 2); - - // Test the Backward function. - moduleMat.Backward(input, input, delta); - REQUIRE(accu(delta) == 160); - REQUIRE(delta.n_rows == 20); -} -*/ - -/** - * Subview index test. - * -TEST_CASE("SubviewIndexTest", "[ANNLayerTest]") -{ - arma::mat outputEnd, outputMid, outputStart, input, delta; - input = arma::linspace(1, 20, 20); - - // Slicing from the initial indices. - Subview moduleStart(1, 0, 9); - arma::mat subStart = arma::linspace(1, 10, 10); - - moduleStart.Forward(input, outputStart); - CheckMatrices(outputStart, subStart); - - // Slicing from the mid indices. - Subview moduleMid(1, 6, 15); - arma::mat subMid = arma::linspace(7, 16, 10); - - moduleMid.Forward(input, outputMid); - CheckMatrices(outputMid, subMid); - - // Slicing from the end indices. - Subview moduleEnd(1, 10, 19); - arma::mat subEnd = arma::linspace(11, 20, 10); - - moduleEnd.Forward(input, outputEnd); - CheckMatrices(outputEnd, subEnd); -} -*/ - -/** - * Subview batch test. - * -TEST_CASE("SubviewBatchTest", "[ANNLayerTest]") -{ - arma::mat output, input, outputCol, outputMat, outputDef; - - // All rows selected. - Subview moduleCol(1, 0, 19); - - // Test with inSize 1. - input = arma::ones(20, 8); - moduleCol.Forward(input, outputCol); - CheckMatrices(outputCol, input); - - // Few rows and columns selected. - Subview moduleMat(4, 3, 6, 0, 2); - - // Test with inSize greater than 1. - moduleMat.Forward(input, outputMat); - output = arma::ones(12, 2); - CheckMatrices(outputMat, output); - - // endCol changed to 3 by default. - Subview moduleDef(4, 1, 6, 0, 4); - - // Test with inSize greater than 1 and endCol >= inSize. - moduleDef.Forward(input, outputDef); - output = arma::ones(24, 2); - CheckMatrices(outputDef, output); -} -*/ - -/** - * Test that the functions that can modify and access the parameters of the - * Subview layer work. - * -TEST_CASE("SubviewLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : inSize, beginRow, endRow, beginCol, endCol. - Subview layer1(1, 2, 3, 4, 5); - Subview layer2(1, 3, 4, 5, 6); - - // Make sure we can get the parameters correctly. - REQUIRE(layer1.InSize() == 1); - REQUIRE(layer1.BeginRow() == 2); - REQUIRE(layer1.EndRow() == 3); - REQUIRE(layer1.BeginCol() == 4); - REQUIRE(layer1.EndCol() == 5); - - // Now modify the parameters to match the second layer. - layer1.BeginRow() = 3; - layer1.EndRow() = 4; - layer1.BeginCol() = 5; - layer1.EndCol() = 6; - - // Now ensure all results are the same. - REQUIRE(layer1.InSize() == layer2.InSize()); - REQUIRE(layer1.BeginRow() == layer2.BeginRow()); - REQUIRE(layer1.EndRow() == layer2.EndRow()); - REQUIRE(layer1.BeginCol() == layer2.BeginCol()); - REQUIRE(layer1.EndCol() == layer2.EndCol()); -} -*/ - -/* - * Simple Reparametrization module test. - * -TEST_CASE("SimpleReparametrizationLayerTest", "[ANNLayerTest]") -{ - arma::mat input, output, delta; - Reparametrization module(5); - - // Test the Forward function. As the mean is zero and the standard - // deviation is small, after multiplying the gaussian sample, the - // output should be small enough. - input = join_cols(arma::ones(5, 1) * -15, - arma::zeros(5, 1)); - module.Forward(input, output); - REQUIRE(arma::accu(output) <= 1e-5); - - // Test the Backward function. - arma::mat gy = arma::zeros(5, 1); - module.Backward(input, gy, delta); - REQUIRE(arma::accu(delta) != 0); // klBackward will be added. -} -*/ - -/** - * Reparametrization module stochastic boolean test. - * -TEST_CASE("ReparametrizationLayerStochasticTest", "[ANNLayerTest]") -{ - arma::mat input, outputA, outputB; - Reparametrization module(5, false); - - input = join_cols(arma::ones(5, 1), - arma::zeros(5, 1)); - - // Test if two forward passes generate same output. - module.Forward(input, outputA); - module.Forward(input, outputB); - - CheckMatrices(outputA, outputB); -} -*/ - -/** - * Reparametrization module includeKl boolean test. - * -TEST_CASE("ReparametrizationLayerIncludeKlTest", "[ANNLayerTest]") -{ - arma::mat input, output, gy, delta; - Reparametrization module(5, true, false); - - input = join_cols(arma::ones(5, 1), - arma::zeros(5, 1)); - module.Forward(input, output); - - // As KL divergence is not included, with the above inputs, the delta - // matrix should be all zeros. - gy = arma::zeros(output.n_rows, output.n_cols); - module.Backward(output, gy, delta); - - REQUIRE(arma::accu(delta) == 0); -} -*/ - -/** - * Jacobian Reparametrization module test. - * -TEST_CASE("JacobianReparametrizationLayerTest", "[ANNLayerTest]") -{ - for (size_t i = 0; i < 5; ++i) - { - const size_t inputElementsHalf = RandInt(2, 10); - - arma::mat input; - input.set_size(inputElementsHalf * 2, 1); - - Reparametrization module(inputElementsHalf, false, false); - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -} -*/ - -/** - * Reparametrization layer numerical gradient test. - * -TEST_CASE("GradientReparametrizationLayerTest", "[ANNLayerTest]") -{ - // Linear function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - input(arma::randu(10, 1)), - target(arma::mat("0")) - { - model = new FFN(); - model->ResetData(input, target); - model->Add(); - model->Add(10, 6); - model->Add(3, false, true, 1); - model->Add(3, 2); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - // REQUIRE(CheckGradient(function) <= 1e-4); -} -*/ - -/** - * Reparametrization layer beta numerical gradient test. - * -TEST_CASE("GradientReparametrizationLayerBetaTest", "[ANNLayerTest]") -{ - // Linear function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - input(arma::randu(10, 2)), - target(arma::mat("0 0")) - { - model = new FFN(); - model->ResetData(input, target); - model->Add(); - model->Add(10, 6); - // Use a value of beta not equal to 1. - model->Add(3, false, true, 2); - model->Add(3, 2); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - // REQUIRE(CheckGradient(function) <= 1e-4); -} -*/ - -/** - * Test that the functions that can access the parameters of the - * Reparametrization layer work. - * -TEST_CASE("ReparametrizationLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : latentSize, stochastic, includeKL, beta. - Reparametrization layer(5, false, false, 2); - - // Make sure we can get the parameters successfully. - REQUIRE(layer.OutputSize() == 5); - REQUIRE(layer.Stochastic() == false); - REQUIRE(layer.IncludeKL() == false); - REQUIRE(layer.Beta() == 2); -} -*/ - -/** - * Simple residual module test. - * -TEST_CASE("SimpleResidualLayerTest", "[ANNLayerTest]") -{ - arma::mat outputA, outputB, input, deltaA, deltaB; - - Sequential* sequential = new Sequential(true); - Residual* residual = new Residual(true); - - Linear* linearA = new Linear(10, 10); - linearA->Parameters().randu(); - linearA->Reset(); - Linear* linearB = new Linear(10, 10); - linearB->Parameters().randu(); - linearB->Reset(); - - // Add the same layers (with the same parameters) to both Sequential and - // Residual object. - sequential->Add(linearA); - sequential->Add(linearB); - - residual->Add(linearA); - residual->Add(linearB); - - // Test the Forward function (pass the same input to both). - input = arma::randu(10, 1); - sequential->Forward(input, outputA); - residual->Forward(input, outputB); - - CheckMatrices(outputA, outputB - input); - - // Test the Backward function (pass the same error to both). - sequential->Backward(input, input, deltaA); - residual->Backward(input, input, deltaB); - - CheckMatrices(deltaA, deltaB - input); - - delete sequential; - delete residual; - delete linearA; - delete linearB; -} -*/ - -/** - * Simple Highway module test. - * -TEST_CASE("SimpleHighwayLayerTest", "[ANNLayerTest]") -{ - arma::mat outputA, outputB, input, deltaA, deltaB; - Sequential* sequential = new Sequential(true); - Highway* highway = new Highway(10, true); - highway->Parameters().zeros(); - highway->Reset(); - - Linear* linearA = new Linear(10, 10); - linearA->Parameters().randu(); - linearA->Reset(); - Linear* linearB = new Linear(10, 10); - linearB->Parameters().randu(); - linearB->Reset(); - - // Add the same layers (with the same parameters) to both Sequential and - // Highway object. - highway->Add(linearA); - highway->Add(linearB); - sequential->Add(linearA); - sequential->Add(linearB); - - // Test the Forward function (pass the same input to both). - input = arma::randu(10, 1); - sequential->Forward(input, outputA); - highway->Forward(input, outputB); - - CheckMatrices(outputB, input * 0.5 + outputA * 0.5); - - delete sequential; - delete highway; - delete linearA; - delete linearB; -} -*/ - -/** - * Test that the function that can access the inSize parameter of the - * Highway layer works. - * -TEST_CASE("HighwayLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : inSize, model. - Highway layer(1, true); - - // Make sure we can get the parameter successfully. - REQUIRE(layer.InSize() == 1); -} -*/ - -// /** -// * Sequential layer numerical gradient test. -// */ -// TEST_CASE("GradientHighwayLayerTest", "[ANNLayerTest]") -// { -// // Linear function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(5, 1)), -// target(arma::mat("0")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add(); -// model->Add(5, 10); - -// highway = new Highway(10); -// highway->Add(10, 10); -// highway->Add(); -// highway->Add(10, 10); -// highway->Add(); - -// model->Add(highway); -// model->Add(10, 2); -// model->Add(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// Highway* highway; -// arma::mat input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -/** - * Sequential layer numerical gradient test. - */ -// TEST_CASE("GradientSequentialLayerTest", "[ANNLayerTest]") -// { -// // Linear function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(10, 1)), -// target(arma::mat("0")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add(); -// model->Add(10, 10); -// sequential = new Sequential(); -// sequential->Add(10, 10); -// sequential->Add(); -// sequential->Add(10, 5); -// sequential->Add(); - -// model->Add(sequential); -// model->Add(5, 2); -// model->Add(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// Sequential* sequential; -// arma::mat input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -// /** -// * WeightNorm layer numerical gradient test. -// */ -// TEST_CASE("GradientWeightNormLayerTest", "[ANNLayerTest]") -// { -// // Linear function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(10, 1)), -// target(arma::mat("0")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add(10, 10); - -// Linear* linear = new Linear(10, 2); -// weightNorm = new WeightNorm(linear); - -// model->Add(weightNorm); -// model->Add(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// WeightNorm* weightNorm; -// arma::mat input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -// /** -// * Test if the WeightNorm layer is able to forward the -// * Forward/Backward/Gradient calls. -// */ -// TEST_CASE("WeightNormRunTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta, error; -// Linear* linear = new Linear(10, 10); - -// WeightNorm module(linear); - -// module.Parameters().randu(); -// module.Reset(); - -// linear->Bias().zeros(); - -// input = arma::zeros(10, 1); -// module.Forward(input, output); - -// // Test the Backward function. -// module.Backward(input, input, delta); - -// REQUIRE(0 == arma::accu(output)); -// REQUIRE(arma::accu(delta) == 0); -// } - -// /** -// * Simple serialization test for layer normalization layer. -// */ -// TEST_CASE("LayerNormSerializationTest", "[ANNLayerTest]") -// { -// LayerNorm<> layer(10); -// ANNLayerSerializationTest(layer); -// } - -/** - * Test that the padding options in Transposed Convolution layer. - * -TEST_CASE("TransposedConvolutionLayerPaddingTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta; - - TransposedConvolution module1(1, 1, 3, 3, 1, 1, 0, 0, 4, 4, 6, 6, "VALID"); - // Test the forward function. - // Valid Should give the same result. - input = arma::linspace(0, 15, 16); - module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); - module1.Reset(); - module1.Forward(input, output); - // Value calculated using tensorflow.nn.conv2d_transpose(). - REQUIRE(arma::accu(output) == 0.0); - - // Test the Backward Function. - module1.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 0.0); - - // Test Valid for non zero padding. - TransposedConvolution module2(1, 1, 3, 3, 2, 2, - std::tuple(0, 0), std::tuple(0, 0), - 2, 2, 5, 5, "VALID"); - // Test the forward function. - input = arma::linspace(0, 3, 4); - module2.Parameters() = arma::mat(25 + 1, 1, arma::fill::zeros); - module2.Parameters()(2) = 8.0; - module2.Parameters()(4) = 6.0; - module2.Parameters()(6) = 4.0; - module2.Parameters()(8) = 2.0; - module2.Reset(); - module2.Forward(input, output); - // Value calculated using torch.nn.functional.conv_transpose2d(). - REQUIRE(arma::accu(output) == 120.0); - - // Test the Backward Function. - module2.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 960.0); - - // Test for same padding type. - TransposedConvolution module3(1, 1, 3, 3, 2, 2, 0, 0, 3, 3, 3, 3, "SAME"); - // Test the forward function. - input = arma::linspace(0, 8, 9); - module3.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); - module3.Reset(); - module3.Forward(input, output); - REQUIRE(arma::accu(output) == 0); - REQUIRE(output.n_rows == input.n_rows); - REQUIRE(output.n_cols == input.n_cols); - - // Test the Backward Function. - module3.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 0.0); - - // Output shape should equal input. - TransposedConvolution module4(1, 1, 3, 3, 1, 1, - std::tuple(2, 2), std::tuple(2, 2), - 5, 5, 5, 5, "SAME"); - // Test the forward function. - input = arma::linspace(0, 24, 25); - module4.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); - module4.Reset(); - module4.Forward(input, output); - REQUIRE(arma::accu(output) == 0); - REQUIRE(output.n_rows == input.n_rows); - REQUIRE(output.n_cols == input.n_cols); - - // Test the Backward Function. - module4.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 0.0); - - TransposedConvolution module5(1, 1, 3, 3, 2, 2, 0, 0, 2, 2, 2, 2, "SAME"); - // Test the forward function. - input = arma::linspace(0, 3, 4); - module5.Parameters() = arma::mat(25 + 1, 1, arma::fill::zeros); - module5.Reset(); - module5.Forward(input, output); - REQUIRE(arma::accu(output) == 0); - REQUIRE(output.n_rows == input.n_rows); - REQUIRE(output.n_cols == input.n_cols); - - // Test the Backward Function. - module5.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 0.0); - - TransposedConvolution module6(1, 1, 4, 4, 1, 1, 1, 1, 5, 5, 5, 5, "SAME"); - // Test the forward function. - input = arma::linspace(0, 24, 25); - module6.Parameters() = arma::mat(16 + 1, 1, arma::fill::zeros); - module6.Reset(); - module6.Forward(input, output); - REQUIRE(arma::accu(output) == 0); - REQUIRE(output.n_rows == input.n_rows); - REQUIRE(output.n_cols == input.n_cols); - - // Test the Backward Function. - module6.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 0.0); -} -*/ - -/** - * Simple test for Lp Pooling layer. - */ -// TEST_CASE("LpMaxPoolingTestCase", "[ANNLayerTest]") -// { -// // For rectangular input to pooling layers. -// arma::mat input = arma::mat(8, 1); -// arma::mat output; -// input.zeros(); -// input(0) = input(6) = 30; -// input(1) = input(7) = 120; -// input(2) = input(4) = 272; -// input(3) = input(5) = 315; -// // Output-Size should be 1 x 2. -// // Square output. -// LpPooling<> module1(4, 2, 2, 2, 2); -// module1.InputHeight() = 2; -// module1.InputWidth() = 4; -// module1.Forward(input, output); -// // Calculated using torch.nn.LPPool2d(). -// REQUIRE(arma::accu(output) - 706.0 == Approx(0.0).margin(2e-5)); -// REQUIRE(output.n_elem == 2); -// -// // For Square input. -// input = arma::mat(16, 1); -// input.zeros(); -// input(0) = 4; -// input(1) = 3; -// input(3) = 12; -// input(7) = 35; -// input(8) = 6; -// input(11) = 7; -// input(12) = 8; -// input(15) = 24; -// // Output-Size should be 2 x 2. -// // Square output. -// LpPooling<> module3(2, 2, 2, 2, 2); -// module3.InputHeight() = 4; -// module3.InputWidth() = 4; -// module3.Forward(input, output); -// // Calculated using torch.nn.LPPool2d(). -// REQUIRE(arma::accu(output) - 77.0 == Approx(0.0).margin(2e-5)); -// REQUIRE(output.n_elem == 4); -// } - -/** - * Test that the functions that can modify and access the parameters of the - * Glimpse layer work. - * -TEST_CASE("GlimpseLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : inSize, size, depth, scale, inputWidth, inputHeight. - Glimpse layer1(1, 2, 3, 4, 5, 6); - Glimpse layer2(1, 2, 3, 4, 6, 7); - - // Make sure we can get the parameters successfully. - REQUIRE(layer1.InputHeight() == 6); - REQUIRE(layer1.InputWidth() == 5); - REQUIRE(layer1.Scale() == 4); - REQUIRE(layer1.Depth() == 3); - REQUIRE(layer1.GlimpseSize() == 2); - REQUIRE(layer1.InSize() == 1); - - // Now modify the parameters to match the second layer. - layer1.InputHeight() = 7; - layer1.InputWidth() = 6; - - // Now ensure that all the results are the same. - REQUIRE(layer1.InputHeight() == layer2.InputHeight()); - REQUIRE(layer1.InputWidth() == layer2.InputWidth()); - REQUIRE(layer1.Scale() == layer2.Scale()); - REQUIRE(layer1.Depth() == layer2.Depth()); - REQUIRE(layer1.GlimpseSize() == layer2.GlimpseSize()); - REQUIRE(layer1.InSize() == layer2.InSize()); -} -*/ - -/** - * Test that the function that can access the stdev parameter of the - * Reinforce Normal layer works. - * -TEST_CASE("ReinforceNormalLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter : stdev. - ReinforceNormal layer(4.0); - - // Make sure we can get the parameter successfully. - REQUIRE(layer.StandardDeviation() == 4.0); -} -*/ - -/* -TEST_CASE("TransposedConvolutionalLayerOptionalParameterTest", "[ANNLayerTest]") -{ - Sequential* decoder = new Sequential(); - - // Check if we can create an object without specifying output. - REQUIRE_NOTHROW(decoder->Add(24, 16, - 5, 5, 1, 1, 0, 0, 10, 10)); - - REQUIRE_NOTHROW(decoder->Add(16, 1, - 15, 15, 1, 1, 1, 1, 14, 14)); - - delete decoder; -} -*/ - -// /** -// * Linear module weight initialization test. -// */ -// TEST_CASE("LinearLayerWeightInitializationTest", "[ANNLayerTest]") -// { -// size_t inSize = 10, outSize = 4; -// Linear<> linear = Linear<>(inSize, outSize); -// linear.Reset(); -// RandomInitialization().Initialize(linear.Weight()); -// linear.Bias().ones(); - -// REQUIRE(std::equal(linear.Weight().begin(), -// linear.Weight().end(), linear.Parameters().begin())); - -// REQUIRE(std::equal(linear.Bias().begin(), -// linear.Bias().end(), linear.Parameters().begin() + inSize * outSize)); - -// REQUIRE(linear.Weight().n_rows == outSize); -// REQUIRE(linear.Weight().n_cols == inSize); -// REQUIRE(linear.Bias().n_rows == outSize); -// REQUIRE(linear.Bias().n_cols == 1); -// REQUIRE(linear.Parameters().n_rows == inSize * outSize + outSize); -// } - -// /** -// * Atrous Convolution module weight initialization test. -// */ -// TEST_CASE("AtrousConvolutionLayerWeightInitializationTest", "[ANNLayerTest]") -// { -// size_t inSize = 2, outSize = 3; -// size_t kernelWidth = 4, kernelHeight = 5; -// AtrousConvolution<> module = AtrousConvolution<>(inSize, outSize, -// kernelWidth, kernelHeight, 6, 7, std::make_tuple(8, 9), -// std::make_tuple(10, 11), 12, 13, 14, 15); -// module.Reset(); -// RandomInitialization().Initialize(module.Weight()); -// module.Bias().ones(); - -// REQUIRE(std::equal(module.Weight().begin(), -// module.Weight().end(), module.Parameters().begin())); - -// REQUIRE(std::equal(module.Bias().begin(), -// module.Bias().end(), module.Parameters().end() - outSize)); - -// REQUIRE(module.Weight().n_rows == kernelWidth); -// REQUIRE(module.Weight().n_cols == kernelHeight); -// REQUIRE(module.Weight().n_slices == inSize * outSize); -// REQUIRE(module.Bias().n_rows == outSize); -// REQUIRE(module.Bias().n_cols == 1); -// REQUIRE(module.Parameters().n_rows -// == (outSize * inSize * kernelWidth * kernelHeight) + outSize); -// } - -/** - * Transposed Convolution module weight initialization test. - * -TEST_CASE("TransposedConvolutionWeightInitializationTest", "[ANNLayerTest]") -{ - size_t inSize = 3, outSize = 3; - size_t kernelWidth = 4, kernelHeight = 4; - TransposedConvolution module = TransposedConvolution(inSize, outSize, - kernelWidth, kernelHeight, 1, 1, 1, 1, 5, 5, 6, 6); - module.Reset(); - RandomInitialization().Initialize(module.Weight()); - module.Bias().ones(); - - REQUIRE(std::equal(module.Weight().begin(), - module.Weight().end(), module.Parameters().begin())); - - REQUIRE(std::equal(module.Bias().begin(), - module.Bias().end(), module.Parameters().end() - outSize)); - - REQUIRE(module.Weight().n_rows == kernelWidth); - REQUIRE(module.Weight().n_cols == kernelHeight); - REQUIRE(module.Weight().n_slices == inSize * outSize); - REQUIRE(module.Bias().n_rows == outSize); - REQUIRE(module.Bias().n_cols == 1); - REQUIRE(module.Parameters().n_rows - == (outSize * inSize * kernelWidth * kernelHeight) + outSize); -} -*/ - -/** - * Simple Test for ChannelShuffle layer. - */ -// TEST_CASE("ChannelShuffleLayerTest", "[ANNLayerTest]") -// { -// arma::mat input1, output1, outputExpected1, outputBackward1; -// ChannelShuffle<> module1(2, 2, 6, 2); -// -// input1 << 1 << 13 << arma::endr -// << 2 << 14 << arma::endr -// << 3 << 15 << arma::endr -// << 4 << 16 << arma::endr -// << 5 << 17 << arma::endr -// << 6 << 18 << arma::endr -// << 7 << 19 << arma::endr -// << 8 << 20 << arma::endr -// << 9 << 21 << arma::endr -// << 10 << 22 << arma::endr -// << 11 << 23 << arma::endr -// << 12 << 24 << arma::endr; -// input1.reshape(24, 1); -// // Value calculated using torch.nn.ChannelShuffle(). -// outputExpected1 << 1 << 17 << arma::endr -// << 2 << 18 << arma::endr -// << 3 << 19 << arma::endr -// << 4 << 20 << arma::endr -// << 13 << 9 << arma::endr -// << 14 << 10 << arma::endr -// << 15 << 11 << arma::endr -// << 16 << 12 << arma::endr -// << 5 << 21 << arma::endr -// << 6 << 22 << arma::endr -// << 7 << 23 << arma::endr -// << 8 << 24 << arma::endr; -// outputExpected1.reshape(24, 1); -// // Check the Forward pass of the layer. -// module1.Forward(input1, output1); -// CheckMatrices(output1, outputExpected1); -// -// // Check the Backward pass of the layer. -// module1.Backward(output1, output1, outputBackward1); -// CheckMatrices(input1, outputBackward1); -// -// } - -/** - * Simple Test for PixelShuffle layer. - */ -// TEST_CASE("PixelShuffleLayerTest", "[ANNLayerTest]") -// { -// arma::mat input1, output1, gy1, g1, outputExpected1, gExpected1; -// arma::mat input2, output2, gy2, g2, outputExpected2, gExpected2; -// PixelShuffle<> module1(2, 2, 2, 4); -// PixelShuffle<> module2(2, 2, 2, 4); -// -// // Input is a single image, of size (2,2) and having 4 channels. -// input1 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 -// << 0 << 0 << arma::endr; -// gy1 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8 -// << 12 << 16 << arma::endr; -// -// // Calculated using torch.nn.PixelShuffle(). -// outputExpected1 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0 -// << 0 << 0 << 0 << 0 << arma::endr; -// gExpected1 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12 -// << 6 << 14 << 8 << 16 << arma::endr; -// -// input1 = input1.t(); -// outputExpected1 = outputExpected1.t(); -// gy1 = gy1.t(); -// gExpected1 = gExpected1.t(); -// -// // Check the Forward pass of the layer. -// module1.Forward(input1, output1); -// CheckMatrices(output1, outputExpected1); -// -// // Check the Backward pass of the layer. -// module1.Backward(input1, gy1, g1); -// CheckMatrices(g1, gExpected1); -// -// // Input is a batch of 2 images, each of size (2,2) and having 4 channels. -// input2 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 -// << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0 -// << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr; -// gy2 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8 -// << 12 << 16 << arma::endr << 17 << 21 << 25 << 29 << 18 << 22 << 26 << 30 -// << 19 << 23 << 27 << 31 << 20 << 24 << 28 << 32 << arma::endr; -// -// // Calculated using torch.nn.PixelShuffle(). -// outputExpected2 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0 -// << 0 << 0 << 0 << 0 << arma::endr << 5 << 0 << 7 << 0 << 0 << 0 << 0 << 0 -// << 6 << 0 << 8 << 0 << 0 << 0 << 0 << 0 << arma::endr; -// gExpected2 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12 -// << 6 << 14 << 8 << 16 << arma::endr << 17 << 25 << 19 << 27 << 21 << 29 -// << 23 << 31 << 18 << 26 << 20 << 28 << 22 << 30 << 24 << 32 << arma::endr; -// -// input2 = input2.t(); -// outputExpected2 = outputExpected2.t(); -// gy2 = gy2.t(); -// gExpected2 = gExpected2.t(); -// -// // Check the Forward pass of the layer. -// module2.Forward(input2, output2); -// CheckMatrices(output2, outputExpected2); -// -// // Check the Backward pass of the layer. -// module2.Backward(input2, gy2, g2); -// CheckMatrices(g2, gExpected2); -// } - -/** - * Test that the function that can access the parameters of the - * PixelShuffle layer works. - */ -// TEST_CASE("PixelShuffleLayerParametersTest", "[ANNLayerTest]") -// { -// // Create the layer using the empty constructor. -// PixelShuffle<> layer; -// -// // Set the different input parameters of the layer. -// layer.UpscaleFactor() = 2; -// layer.InputHeight() = 2; -// layer.InputWidth() = 2; -// layer.InputChannels() = 4; -// -// // Make sure we can get the parameters successfully. -// REQUIRE(layer.UpscaleFactor() == 2); -// REQUIRE(layer.InputHeight() == 2); -// REQUIRE(layer.InputWidth() == 2); -// REQUIRE(layer.InputChannels() == 4); -// -// arma::mat input, output; -// // Input is a batch of 2 images, each of size (2,2) and having 4 channels. -// input << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 -// << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0 -// << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr; -// input = input.t(); -// layer.Forward(input, output); -// -// // Check whether output parameters are returned correctly. -// REQUIRE(layer.OutputHeight() == 4); -// REQUIRE(layer.OutputWidth() == 4); -// REQUIRE(layer.OutputChannels() == 1); -// } - -// /** -// * Simple Test for SpatialDropout layer. -// */ -// TEST_CASE("SpatialDropoutLayerTest", "[ANNLayerTest]") -// { -// arma::mat input, output, gy, g, temp; -// arma::mat outputsExpected = arma::zeros(8, 12); -// arma::mat gsExpected = arma::zeros(8, 12); - -// // Set the seed to a random value. -// arma::arma_rng::set_seed_random(); -// SpatialDropout<> module(3, 0.2); - -// // Input is a batch of 2 images, each of size (2,2) and having 4 channels. -// input = { 0.4963, 0.0885, 0.7682, 0.1320, 0.3074, 0.4901, 0.6341, 0.8964, -// 0.4556, 0.3489, 0.6323, 0.4017 }; -// -// gy = { 1, 3, 2, 4, 5, 7, 6, 8, 9, 11, 10, 12 }; -// -// // Following values have been calculated using torch.nn.Dropout2d(p=0.2). -// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; -// outputsExpected.row(0) = temp; -// temp = { 0, 0, 0, 0, 0.3842, 0.6126, 0.7926, 1.1205, 0.5695, 0.4361, 0.7904, -// 0.5021 }; -// outputsExpected.row(1) = temp; -// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0, 0, 0, 0, 0.5695, 0.4361, -// 0.7904, 0.5021 }; -// outputsExpected.row(2) = temp; -// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0.3842, 0.6126, 0.7926, 1.1205, 0, -// 0, 0, 0 }; -// outputsExpected.row(3) = temp; -// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 0.5695, 0.4361, 0.7904, 0.5021 }; -// outputsExpected.row(4) = temp; -// temp = { 0, 0, 0, 0, 0.3842, 0.6126, 0.7926, 1.1205, 0, 0, 0, 0 }; -// outputsExpected.row(5) = temp; -// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0, 0, 0, 0, 0, 0, 0, 0 }; -// outputsExpected.row(6) = temp; -// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0.3842, 0.6126, 0.7926, 1.1205, -// 0.5695, 0.4361, 0.7904, 0.5021 }; -// outputsExpected.row(7) = temp; -// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; -// gsExpected.row(0) = temp; -// temp = { 0, 0, 0, 0, 6.2500, 8.7500, 7.5000, 10.0000, 11.2500, 13.7500, -// 12.5000, 15.0000 }; -// gsExpected.row(1) = temp; -// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 0, 0, 0, 0, 11.2500, 13.7500, -// 12.5000, 15.0000 }; -// gsExpected.row(2) = temp; -// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 6.2500, 8.7500, 7.5000, 10.0000, 0, -// 0, 0, 0 }; -// gsExpected.row(3) = temp; -// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 11.2500, 13.7500, 12.5000, 15.0000 }; -// gsExpected.row(4) = temp; -// temp = { 0, 0, 0, 0, 6.2500, 8.7500, 7.5000, 10.0000, 0, 0, 0, 0 }; -// gsExpected.row(5) = temp; -// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 0, 0, 0, 0, 0, 0, 0, 0 }; -// gsExpected.row(6) = temp; -// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 6.2500, 8.7500, 7.5000, 10.0000, -// 11.2500, 13.7500, 12.5000, 15.0000 }; -// gsExpected.row(7) = temp; - -// input = input.t(); -// gy = gy.t(); -// outputsExpected = outputsExpected.t(); -// gsExpected = gsExpected.t(); - -// // Compute the Forward and Backward passes and store the results. -// module.Forward(input, output); -// module.Backward(input, gy, g); - -// // Check through all possible cases, to find a match and then compare results. -// for (size_t i = 0; i < outputsExpected.n_cols; ++i) -// { -// if (arma::approx_equal(outputsExpected.col(i), output, "absdiff", 1e-1)) -// { -// // Check the correctness of the Forward pass of the layer. -// CheckMatrices(output, outputsExpected.col(i), 1e-1); -// // Check the correctness of the Backward pass of the layer. -// CheckMatrices(g, gsExpected.col(i), 1e-1); -// } -// } - -// // Check if the output is same as input when using deterministic mode. -// module.Deterministic() = true; -// output.clear(); -// module.Forward(input, output); -// CheckMatrices(output, input, 1e-1); -// } - -// /** -// * Test that the function that can access the parameters of the -// * SpatialDropout layer works. -// */ -// TEST_CASE("SpatialDropoutLayerParametersTest", "[ANNLayerTest]") -// { -// // Create the layer using the empty constructor. -// SpatialDropout<> layer; - -// // Set the input parameters. -// layer.Size() = 3; -// layer.Ratio(0.2); - -// // Check whether the input parameters have been set correctly. -// REQUIRE(layer.Size() == 3); -// REQUIRE(layer.Ratio() == 0.2); -// } - -/** - * Simple Positional Encoding layer test. - * -TEST_CASE("SimplePositionalEncodingTest", "[ANNLayerTest]") -{ - const size_t seqLength = 5; - const size_t embedDim = 4; - const size_t batchSize = 2; - - arma::mat input = arma::randu(embedDim * seqLength, batchSize); - arma::mat gy = 0.01 * arma::randu(embedDim * seqLength, batchSize); - arma::mat output, g; - - PositionalEncoding module(embedDim, seqLength); - - // Check Forward function. - module.Forward(input, output); - arma::mat pe = output - input; - CheckMatrices(arma::mean(pe, 1), module.Encoding()); - - // Check Backward function. - module.Backward(input, gy, g); - REQUIRE(std::equal(gy.begin(), gy.end(), g.begin())); -} -*/ - -/** - * Jacobian test for Positional Encoding layer. - * -TEST_CASE("JacobianPositionalEncodingTest", "[ANNLayerTest]") -{ - for (size_t i = 0; i < 5; ++i) - { - const size_t embedDim = 4; - const size_t seqLength = RandInt(5, 10); - arma::mat input; - input.set_size(embedDim * seqLength, 1); - - PositionalEncoding module(embedDim, seqLength); - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -} -*/ - -/** - * Simple Multihead Attention test. - * -TEST_CASE("SimpleMultiheadAttentionTest", "[ANNLayerTest]") -{ - size_t tLen = 5; - size_t sLen = tLen; - size_t embedDim = 4; - size_t numHeads = 2; - size_t bsz = 3; - - arma::mat query = 0.1 * arma::randu(embedDim * tLen, bsz); - arma::mat output; - - arma::mat attnMask = arma::zeros(tLen, sLen); - for (size_t i = 0; i < tLen; ++i) - { - for (size_t j = 0; j < sLen; ++j) - { - if (i < j) - attnMask(i, j) = std::numeric_limits::lowest(); - } - } - - arma::mat keyPaddingMask = arma::zeros(1, sLen); - keyPaddingMask(sLen - 1) = std::numeric_limits::lowest(); - - MultiheadAttention module(tLen, sLen, embedDim, numHeads); - module.AttentionMask() = attnMask; - module.KeyPaddingMask() = keyPaddingMask; - module.Reset(); - module.Parameters().randu(); - - // Forward test. - arma::mat input = arma::join_cols(arma::join_cols(query, query), query); - - module.Forward(input, output); - REQUIRE(output.n_rows == embedDim * tLen); - REQUIRE(output.n_cols == bsz); - - // Backward test. - arma::mat gy = 0.01 * arma::randu(embedDim * tLen, bsz); - arma::mat g; - module.Backward(input, gy, g); - REQUIRE(g.n_rows == input.n_rows); - REQUIRE(g.n_cols == input.n_cols); - - // Gradient test. - arma::mat error = 0.05 * arma::randu(embedDim * tLen, bsz); - arma::mat gradient; - module.Gradient(input, error, gradient); - REQUIRE(gradient.n_rows == module.Parameters().n_rows); - REQUIRE(gradient.n_cols == module.Parameters().n_cols); -} -*/ - -/** - * Jacobian MultiheadAttention module test. - * -TEST_CASE("JacobianMultiheadAttentionTest", "[ANNLayerTest]") -{ - // Check when query = key = value. - for (size_t i = 0; i < 5; ++i) - { - const size_t tgtSeqLen = 2; - const size_t embedDim = 4; - const size_t nHeads = 2; - const size_t batchSize = 1; - - arma::mat query = arma::randu(embedDim * tgtSeqLen, batchSize); - arma::mat input = arma::join_cols(arma::join_cols(query, query), query); - - MultiheadAttention module(tgtSeqLen, tgtSeqLen, embedDim, nHeads); - module.Parameters().randu(); - - double error = CustomJacobianTest(module, input); - REQUIRE(error <= 1e-5); - } - - // Check when key = value. - for (size_t i = 0; i < 5; ++i) - { - const size_t tgtSeqLen = 2; - const size_t srcSeqLen = RandInt(2, 5); - const size_t embedDim = 4; - const size_t nHeads = 2; - const size_t batchSize = 1; - - arma::mat query = arma::randu(embedDim * tgtSeqLen, batchSize); - arma::mat key = 0.091 * arma::randu(embedDim * srcSeqLen, batchSize); - arma::mat input = arma::join_cols(arma::join_cols(query, key), key); - - MultiheadAttention module(tgtSeqLen, srcSeqLen, embedDim, nHeads); - module.Parameters().randu(); - - double error = CustomJacobianTest(module, input); - REQUIRE(error <= 1e-5); - } - - // Check when query, key and value are not same. - for (size_t i = 0; i < 5; ++i) - { - const size_t tgtSeqLen = 2; - const size_t srcSeqLen = RandInt(2, 5); - const size_t embedDim = 4; - const size_t nHeads = 2; - const size_t batchSize = 1; - - arma::mat query = arma::randu(embedDim * tgtSeqLen, batchSize); - arma::mat key = 0.091 * arma::randu(embedDim * srcSeqLen, batchSize); - arma::mat value = 0.045 * arma::randu(embedDim * srcSeqLen, batchSize); - arma::mat input = arma::join_cols(arma::join_cols(query, key), value); - - MultiheadAttention module(tgtSeqLen, srcSeqLen, embedDim, nHeads); - module.Parameters().randu(); - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -} -*/ - -/** - * Numerical gradient test for MultiheadAttention layer. - * -TEST_CASE("GradientMultiheadAttentionTest", "[ANNLayerTest]") -{ - struct GradientFunction - { - GradientFunction() : - tgtSeqLen(2), - srcSeqLen(2), - embedDim(4), - nHeads(2), - vocabSize(5), - batchSize(2) - { - input = arma::randu(embedDim * (tgtSeqLen + 2 * srcSeqLen), batchSize); - target = arma::zeros(vocabSize, batchSize); - for (size_t i = 0; i < target.n_elem; ++i) - { - const size_t label = RandInt(1, vocabSize); - target(i) = label; - } - - attnMask = arma::zeros(tgtSeqLen, srcSeqLen); - for (size_t i = 0; i < tgtSeqLen; ++i) - { - for (size_t j = 0; j < srcSeqLen; ++j) - { - if (i < j) - attnMask(i, j) = std::numeric_limits::lowest(); - } - } - - keyPaddingMask = arma::zeros(1, srcSeqLen); - keyPaddingMask(srcSeqLen - 1) = std::numeric_limits::lowest(); - - model = new FFN(); - model->ResetData(input, target); - // attnModule = new MultiheadAttention(tgtSeqLen, srcSeqLen, embedDim, - // nHeads); - // attnModule->AttentionMask() = attnMask; - // attnModule->KeyPaddingMask() = keyPaddingMask; - // model->Add(attnModule); - model->Add(tgtSeqLen, srcSeqLen, embedDim, nHeads, - attnMask, keyPaddingMask); - model->Add(embedDim * tgtSeqLen, vocabSize); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, batchSize); - model->Gradient(model->Parameters(), 0, gradient, batchSize); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - // MultiheadAttention* attnModule; - - arma::mat input, target, attnMask, keyPaddingMask; - const size_t tgtSeqLen; - const size_t srcSeqLen; - const size_t embedDim; - const size_t nHeads; - const size_t vocabSize; - const size_t batchSize; - } function; - - REQUIRE(CheckGradient(function) <= 3e-06); -} -*/ - -/** - * Simple tests for instance normalization layer. - * -TEST_CASE("InstanceNormLayerTest", "[ANNLayerTest]") -{ - arma::mat input, result, output, delta, deltaExpected; - arma::mat runningMean, runningVar; - - // Represents 2 images, each having 3 channels, and shape (3,2). - input << 1 << 19 << arma::endr - << 2 << 20 << arma::endr - << 3 << 21 << arma::endr - << 4 << 22 << arma::endr - << 5 << 23 << arma::endr - << 6 << 24 << arma::endr - << 7 << 25 << arma::endr - << 8 << 26 << arma::endr - << 9 << 27 << arma::endr - << 10 << 28 << arma::endr - << 11 << 29 << arma::endr - << 12 << 30 << arma::endr - << 13 << 31 << arma::endr - << 14 << 32 << arma::endr - << 15 << 33 << arma::endr - << 16 << 34 << arma::endr - << 17 << 35 << arma::endr - << 18 << 36 << arma::endr; - - // Output calculated using torch.nn.InstanceNorm2d(). - result << -1.4638 << -1.4638 << arma::endr - << -0.8783 << -0.8783 << arma::endr - << -0.2928 << -0.2928 << arma::endr - << 0.2928 << 0.2928 << arma::endr - << 0.8783 << 0.8783 << arma::endr - << 1.4638 << 1.4638 << arma::endr - << -1.4638 << -1.4638 << arma::endr - << -0.8783 << -0.8783 << arma::endr - << -0.2928 << -0.2928 << arma::endr - << 0.2928 << 0.2928 << arma::endr - << 0.8783 << 0.8783 << arma::endr - << 1.4638 << 1.4638 << arma::endr - << -1.4638 << -1.4638 << arma::endr - << -0.8783 << -0.8783 << arma::endr - << -0.2928 << -0.2928 << arma::endr - << 0.2928 << 0.2928 << arma::endr - << 0.8783 << 0.8783 << arma::endr - << 1.4638 << 1.4638 << arma::endr; - - // Calculated using torch.nn.InstanceNorm2d(). - deltaExpected << 1.8367 << 1.8367 << arma::endr - << 0.3967 << 0.3967 << arma::endr - << 0.0147 << 0.0147 << arma::endr - <<-0.0147 << -0.0147 << arma::endr - <<-0.3967 << -0.3967 << arma::endr - <<-1.8367 << -1.8367 << arma::endr - << 1.8367 << 1.8367 << arma::endr - << 0.3967 << 0.3967 << arma::endr - << 0.0147 << 0.0147 << arma::endr - <<-0.0147 << -0.0147 << arma::endr - <<-0.3967 << -0.3967 << arma::endr - <<-1.8367 << -1.8367 << arma::endr - << 1.8367 << 1.8367 << arma::endr - << 0.3967 << 0.3967 << arma::endr - << 0.0147 << 0.0147 << arma::endr - <<-0.0147 << -0.0147 << arma::endr - <<-0.3967 << -0.3967 << arma::endr - <<-1.8367 << -1.8367 << arma::endr; - - // Check Forward and Backward pass in non-deterministic mode. - InstanceNorm<> module(3, input.n_cols, 1e-5, false, 0.1); - output.zeros(arma::size(input)); - module.Forward(input, output); - CheckMatrices(output, result, 1e-1); - - module.Backward(input, output, delta); - CheckMatrices(delta, deltaExpected, 1e-1); - - runningMean = arma::mat(3, 1); - runningVar = arma::mat(3, 1); - runningMean(0) = 1.2500; - runningMean(1) = 1.8500; - runningMean(2) = 2.4500; - runningVar(0) = 1.2500; - runningVar(1) = 1.2500; - runningVar(2) = 1.2500; - - CheckMatrices(runningMean, module.TrainingMean(), 1e-1); - CheckMatrices(runningVar, module.TrainingVariance(), 1e-1); - - // Check Forward pass in deterministic mode. - InstanceNorm<> module1(3, input.n_cols, 1e-5, false, 0.1); - module1.Deterministic() = true; - output.zeros(arma::size(input)); - module1.Forward(input, output); - - // Calculated using torch.nn.InstanceNorm2d(). - result << 1.0000 << 18.9999 << arma::endr - << 2.0000 << 19.9999 << arma::endr - << 3.0000 << 20.9999 << arma::endr - << 4.0000 << 21.9999 << arma::endr - << 5.0000 << 22.9999 << arma::endr - << 6.0000 << 23.9999 << arma::endr - << 7.0000 << 24.9999 << arma::endr - << 8.0000 << 25.9999 << arma::endr - << 9.0000 << 26.9999 << arma::endr - << 10.0000 << 27.9999 << arma::endr - << 10.9999 << 28.9999 << arma::endr - << 11.9999 << 29.9999 << arma::endr - << 12.9999 << 30.9998 << arma::endr - << 13.9999 << 31.9998 << arma::endr - << 14.9999 << 32.9998 << arma::endr - << 15.9999 << 33.9998 << arma::endr - << 16.9999 << 34.9998 << arma::endr - << 17.9999 << 35.9998 << arma::endr; - - CheckMatrices(output, result, 1e-1); -} -*/ - -/** - * Test that the functions that can access the parameters of the - * Instance Norm layer work. - * -TEST_CASE("InstanceNormLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : size, eps. - InstanceNorm<> layer(7, 0, 1e-3); - - // Make sure we can get the parameters successfully. - REQUIRE(layer.InputSize() == 7); - REQUIRE(layer.Epsilon() == 1e-3); - - arma::mat runningMean(7, 1, arma::fill::randn); - arma::mat runningVariance(7, 1, arma::fill::randn); - - layer.TrainingVariance() = runningVariance; - layer.TrainingMean() = runningMean; - CheckMatrices(layer.TrainingVariance(), runningVariance); - CheckMatrices(layer.TrainingMean(), runningMean); -} -*/ - -/** - * Instance Norm layer numerical gradient test. - * -TEST_CASE("GradientInstanceNormLayerTest", "[ANNLayerTest]") -{ - // Add function gradient instantiation. - // To make this test robust, check it ten times. - bool pass = false; - for (size_t trial = 0; trial < 10; trial++) - { - struct GradientFunction - { - GradientFunction() - { - input = arma::randn(16, 1024); - arma::mat target; - target.ones(1, 1024); - - model = new FFN(); - model->ResetData(input, target); - model->Add >(); - model->Add >(1, 2, 3, 3, 1, 1, 0, 0, 4, 4); - model->Add > (2, 1024); - model->Add >(2 * 2 * 2, 2); - model->Add >(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1024, false); - model->Gradient(model->Parameters(), 0, gradient, 1024); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - double gradient = CheckGradient(function); - if (gradient < 1e-1) - { - pass = true; - break; - } - } - - REQUIRE(pass); -} -*/ diff --git a/src/mlpack/tests/ann/not_adapted/ann_layer_test.cpp b/src/mlpack/tests/ann/not_adapted/ann_layer_test.cpp index c1d4a242fd..cfd9ec0464 100644 --- a/src/mlpack/tests/ann/not_adapted/ann_layer_test.cpp +++ b/src/mlpack/tests/ann/not_adapted/ann_layer_test.cpp @@ -386,27 +386,6 @@ TEST_CASE("ConstantLayerParametersTest", "[ANNLayerTest]") // REQUIRE(CheckGradient(function) <= 1e-4); // } -// /** -// * Jacobian negative log likelihood module test. -// */ -// TEST_CASE("JacobianNegativeLogLikelihoodLayerTest", "[ANNLayerTest]") -// { -// for (size_t i = 0; i < 5; ++i) -// { -// NegativeLogLikelihood module; -// const size_t inputElements = RandInt(5, 100); -// arma::mat input; -// RandomInitialization init(0, 1); -// init.Initialize(input, inputElements, 1); - -// arma::mat target(1, 1); -// target(0) = RandInt(0, inputElements - 2); - -// double error = JacobianPerformanceTest(module, input, target); -// REQUIRE(error <= 1e-5); -// } -// } - /** * Jacobian LeakyReLU module test. * @@ -1848,285 +1827,6 @@ TEST_CASE("SimpleJoinLayerTest", "[ANNLayerTest]") // boost::apply_visitor(DeleteVisitor(), layer); // } -/** - * Simple concat module test. - */ -TEST_CASE("SimpleConcatLayerTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta, error; - - Linear* moduleA = new Linear(10); - moduleA->InputDimensions() = std::vector({ 10 }); - moduleA->ComputeOutputDimensions(); - arma::mat weightsA(moduleA->WeightSize(), 1); - moduleA->SetWeights((double*) weightsA.memptr()); - moduleA->Parameters().randu(); - - Linear* moduleB = new Linear(10); - moduleB->InputDimensions() = std::vector({ 10 }); - moduleB->ComputeOutputDimensions(); - arma::mat weightsB(moduleB->WeightSize(), 1); - moduleB->SetWeights((double*) weightsB.memptr()); - moduleB->Parameters().randu(); - - Concat module; - module.Add(moduleA); - module.Add(moduleB); - module.InputDimensions() = std::vector({ 10 }); - module.ComputeOutputDimensions(); - - // Test the Forward function. - input = arma::zeros(10, 1); - output.set_size(module.OutputSize(), 1); - module.Forward(input, output); - - const double sumModuleA = arma::accu( - moduleA->Parameters().submat( - 100, 0, moduleA->Parameters().n_elem - 1, 0)); - const double sumModuleB = arma::accu( - moduleB->Parameters().submat( - 100, 0, moduleB->Parameters().n_elem - 1, 0)); - REQUIRE(sumModuleA + sumModuleB == - Approx(arma::accu(output.col(0))).epsilon(1e-5)); - - // Test the Backward function. - error = arma::zeros(20, 1); - delta.set_size(input.n_rows, input.n_cols); - module.Backward(input, error, delta); - REQUIRE(arma::accu(delta) == 0); -} - -/** - * Test to check Concat layer along different axes. - */ -TEST_CASE("ConcatAlongAxisTest", "[ANNLayerTest]") -{ - arma::mat output, input, error, outputA, outputB; - size_t inputWidth = 4, inputHeight = 4, inputChannel = 2; - size_t outputWidth, outputHeight, outputChannel = 2; - size_t kW = 3, kH = 3; - size_t batch = 1; - - // Using Convolution<> layer as inout to Concat<> layer. - // Compute the output shape of convolution layer. - outputWidth = (inputWidth - kW) + 1; - outputHeight = (inputHeight - kH) + 1; - - input = arma::ones(inputWidth * inputHeight * inputChannel, batch); - - Convolution* moduleA = new Convolution(outputChannel, kW, kH, 1, 1, 0, 0); - Convolution* moduleB = new Convolution(outputChannel, kW, kH, 1, 1, 0, 0); - - moduleA->InputDimensions() = std::vector({ inputWidth, inputHeight }); - moduleA->ComputeOutputDimensions(); - arma::mat weightsA(moduleA->WeightSize(), 1); - moduleA->SetWeights((double*) weightsA.memptr()); - moduleA->Parameters().randu(); - - moduleB->InputDimensions() = std::vector({ inputWidth, inputHeight }); - moduleB->ComputeOutputDimensions(); - arma::mat weightsB(moduleB->WeightSize(), 1); - moduleB->SetWeights((double*) weightsB.memptr()); - moduleB->Parameters().randu(); - - // Compute output of each layer. - outputA.set_size(moduleA->OutputSize(), 1); - outputB.set_size(moduleB->OutputSize(), 1); - moduleA->Forward(input, outputA); - moduleB->Forward(input, outputB); - - arma::cube A(outputA.memptr(), outputWidth, outputHeight, outputChannel); - arma::cube B(outputB.memptr(), outputWidth, outputHeight, outputChannel); - - error = arma::ones(outputWidth * outputHeight * outputChannel * 2, 1); - - for (size_t axis = 0; axis < 3; ++axis) - { - size_t x = 1, y = 1, z = 1; - arma::cube calculatedOut; - if (axis == 0) - { - calculatedOut.set_size(2 * outputWidth, outputHeight, outputChannel); - for (size_t i = 0; i < A.n_slices; ++i) - { - arma::mat aMat = A.slice(i); - arma::mat bMat = B.slice(i); - calculatedOut.slice(i) = arma::join_cols(aMat, bMat); - } - x = 2; - } - if (axis == 1) - { - calculatedOut.set_size(outputWidth, 2 * outputHeight, outputChannel); - for (size_t i = 0; i < A.n_slices; ++i) - { - arma::mat aMat = A.slice(i); - arma::mat bMat = B.slice(i); - calculatedOut.slice(i) = arma::join_rows(aMat, bMat); - } - y = 2; - } - if (axis == 2) - { - calculatedOut = arma::join_slices(A, B); - z = 2; - } - - // Compute output of Concat<> layer. - Concat module(axis); - module.Add(moduleA); - module.Add(moduleB); - module.InputDimensions() = std::vector({ inputWidth, inputHeight }); - module.ComputeOutputDimensions(); - output.set_size(module.OutputSize(), 1); - module.Forward(input, output); - arma::cube concatOut(output.memptr(), x * outputWidth, - y * outputHeight, z * outputChannel); - - // Verify if the output reshaped to cubes are similar. - CheckMatrices(concatOut, calculatedOut, 1e-12); - - // Ensure that the child layers don't get deleted when `module` is - // deallocated. - module.Network().clear(); - } - - delete moduleA; - delete moduleB; -} - -/** - * Test that the function that can access the axis parameter of the - * Concat layer works. - */ -TEST_CASE("ConcatLayerParametersTest", "[ANNLayerTest]") -{ - Concat layer(2); - - // Make sure we can get the parameters successfully. - REQUIRE(layer.Axis() == 2); -} - -/** - * Concat layer numerical gradient test. - */ -TEST_CASE("GradientConcatLayerTest", "[ANNLayerTest]") -{ - // Concat function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - input(arma::randu(10, 1)), - target(arma::mat("0")) - { - model = new FFN(); - model->ResetData(input, target); - model->Add(10); - - concat = new Concat(); - concat->Add(5); - concat->Add(5); - model->Add(concat); - model->Add(2); - - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - Concat* concat; - arma::mat input, target; - } function; - - REQUIRE(CheckGradient(function) <= 1e-4); -} - -/** - * Simple concatenate module test. - */ -TEST_CASE("SimpleConcatenateLayerTest", "[ANNLayerTest]") -{ - arma::mat input = arma::ones(5, 1); - arma::mat output, delta; - - Concatenate module; - module.Concat() = arma::ones(5, 1) * 0.5; - module.InputDimensions() = std::vector({ 5 }); - module.ComputeOutputDimensions(); - - // Test the Forward function. - output.set_size(module.OutputSize(), 1); - module.Forward(input, output); - - REQUIRE(arma::accu(output) == 7.5); - - // Test the Backward function. - delta.set_size(5, 1); - module.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 5); -} - -/** - * Concatenate layer numerical gradient test. - */ -TEST_CASE("GradientConcatenateLayerTest", "[ANNLayerTest]") -{ - // Concatenate function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - input(arma::randu(10, 1)), - target(arma::mat("0")) - { - model = new FFN(); - model->ResetData(input, target); - model->Add(5); - - arma::mat concat = arma::ones(5, 1); - // concatenate = new Concatenate(); - // concatenate->Concat() = concat; - // model->Add(concatenate); - model->Add(concat); - - model->Add(5); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - Concatenate* concatenate; - arma::mat input, target; - } function; - - REQUIRE(CheckGradient(function) <= 1e-4); -} - /** * Simple lookup module test. * @@ -2239,69 +1939,6 @@ TEST_CASE("LookupLayerParametersTest", "[ANNLayerTest]") } */ -/** - * Simple Softmax module test. - */ -TEST_CASE("SimpleSoftmaxLayerTest", "[ANNLayerTest]") -{ - arma::mat input, output, gy, g; - Softmax module; - - // Test the forward function. - input = arma::mat("1.7; 3.6"); - module.Forward(input, output); - REQUIRE(arma::accu(arma::abs(arma::mat("0.130108; 0.869892") - output)) == - Approx(0.0).margin(1e-4)); - - // Test the backward function. - gy = arma::zeros(input.n_rows, input.n_cols); - gy(0) = 1; - module.Backward(output, gy, g); - REQUIRE(arma::accu(arma::abs(arma::mat("0.11318; -0.11318") - g)) == - Approx(0.0).margin(1e-04)); -} - -/** - * Softmax layer numerical gradient test. - */ -TEST_CASE("GradientSoftmaxTest", "[ANNLayerTest]") -{ - // Softmax function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - input(arma::randu(10, 1)), - target(arma::mat("1; 0")) - { - model = new FFN; - model->ResetData(input, target); - model->Add(10); - model->Add(); - model->Add(2); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - REQUIRE(CheckGradient(function) <= 1e-4); -} - /** * Simple test for the NearestInterpolation layer * @@ -4058,155 +3695,6 @@ TEST_CASE("TransposedConvolutionLayerPaddingTest", "[ANNLayerTest]") // REQUIRE(output.n_elem == 4); // } -/** - * Simple test for AddMerge layer. - */ -TEST_CASE("AddMergeTestCase", "[ANNLayerTest]") -{ - // For rectangular input to pooling layers. - arma::mat input = arma::mat(28, 1); - input.zeros(); - input(0) = input(16) = 1; - input(1) = input(17) = 2; - input(2) = input(18) = 3; - input(3) = input(19) = 4; - input(4) = input(20) = 5; - input(5) = input(23) = 6; - input(6) = input(24) = 7; - input(14) = input(25) = 8; - input(15) = input(26) = 9; - - AddMerge module1; - module1.Add(2, 2, 2, 2, false); - module1.Add(2, 2, 2, 2, false); - - AddMerge module2; - module2.Add(2, 2, 2, 2, true); - module2.Add(2, 2, 2, 2, true); - - module1.InputDimensions() = std::vector({ 7, 4 }); - module1.ComputeOutputDimensions(); - module2.InputDimensions() = std::vector({ 7, 4 }); - module2.ComputeOutputDimensions(); - - // Calculated using torch.nn.MeanPool2d(). - arma::mat result1, result2; - result1 << 1.5000 << 8.5000 << arma::endr - << 3.5000 << 8.0000 << arma::endr - << 5.5000 << 12.0000 << arma::endr - << 7.0000 << 5.0000 << arma::endr; - - result2 << 1.5000 << 8.5000 << arma::endr - << 3.5000 << 8.0000 << arma::endr - << 5.5000 << 12.0000 << arma::endr; - - arma::mat output1, output2; - output1.set_size(8, 1); - output2.set_size(6, 1); - module1.Forward(input, output1); - REQUIRE(arma::accu(output1) == 51.0); - module2.Forward(input, output2); - REQUIRE(arma::accu(output2) == 39.0); - output1.reshape(4, 2); - output2.reshape(3, 2); - CheckMatrices(output1, result1, 1e-1); - CheckMatrices(output2, result2, 1e-1); - - arma::mat prevDelta1, prevDelta2; - prevDelta1 << 3.6000 << -0.9000 << arma::endr - << 3.6000 << -0.9000 << arma::endr - << 3.6000 << -0.9000 << arma::endr - << 3.6000 << -0.9000 << arma::endr; - - prevDelta2 << 3.6000 << -0.9000 << arma::endr - << 3.6000 << -0.9000 << arma::endr - << 3.6000 << -0.9000 << arma::endr; - arma::mat delta1, delta2; - delta1.set_size(28, 1); - delta2.set_size(28, 1); - prevDelta1.reshape(8, 1); - prevDelta2.reshape(6, 1); - module1.Backward(input, prevDelta1, delta1); - REQUIRE(arma::accu(delta1) == Approx(21.6).epsilon(1e-3)); - module2.Backward(input, prevDelta2, delta2); - REQUIRE(arma::accu(delta2) == Approx(16.2).epsilon(1e-3)); -} - -/** - * Complex test for AddMerge layer. - * This test includes: - * 1. AddMerge layer inside the AddMerge layer. - * 2. Batch Size > 1. - * 3. AddMerge layer with single child layer. - */ -TEST_CASE("AddMergeAdvanceTestCase", "[ANNLayerTest]") -{ - AddMerge r; - AddMerge* r2 = new AddMerge(); - r2->Add(5); - r.Add(5); - r.Add(r2); - r.InputDimensions() = std::vector({ 5 }); - r.ComputeOutputDimensions(); - arma::mat rParams(r.WeightSize(), 1); - r.SetWeights((double*) rParams.memptr()); - r.Network()[0]->Parameters().fill(2.0); - ((AddMerge*) r.Network()[1])->Network()[0]->Parameters().fill(-1.0); - - Linear l(5); - l.InputDimensions() = std::vector({ 5 }); - l.ComputeOutputDimensions(); - arma::mat lParams(l.WeightSize(), 1); - l.SetWeights((double*) lParams.memptr()); - l.Parameters().fill(1.0); - - arma::mat input(arma::randn(5, 10)); - arma::mat output1, output2; - output1.set_size(5, 10); - output2.set_size(5, 10); - - r.Forward(input, output1); - l.Forward(input, output2); - - CheckMatrices(output1, output2, 1e-3); - - arma::mat delta1, delta2; - delta1.set_size(5, 10); - delta2.set_size(5, 10); - r.Backward(input, output1, delta1); - l.Backward(input, output2, delta2); - - CheckMatrices(output1, output2, 1e-3); -} - -/** - * Simple test for Identity layer. - */ -TEST_CASE("IdentityTestCase", "[ANNLayerTest]") -{ - // For rectangular input to pooling layers. - arma::mat input = arma::mat(12, 1, arma::fill::randn); - arma::mat output; - // Output-Size should be 4 x 3. - output.set_size(12, 1); - - Identity module1; - module1.InputDimensions() = std::vector({ 4, 3 }); - module1.ComputeOutputDimensions(); - module1.Forward(input, output); - CheckMatrices(output, input, 1e-1); - REQUIRE(output.n_elem == 12); - REQUIRE(output.n_cols == 1); - REQUIRE(input.memptr() != output.memptr()); - - arma::mat prevDelta = arma::mat(12, 1, arma::fill::randn); - arma::mat delta; - delta.set_size(12, 1); - module1.Backward(input, prevDelta, delta); - CheckMatrices(delta, prevDelta, 1e-1); - REQUIRE(delta.memptr() != prevDelta.memptr()); -} - /** * Test that the functions that can modify and access the parameters of the * Glimpse layer work. From 78f519b91e6389f13ec3b3c47912f7efea346060 Mon Sep 17 00:00:00 2001 From: AdarshSantoria Date: Wed, 1 Mar 2023 10:56:39 +0530 Subject: [PATCH 83/88] fix code --- .../ann/layer/parametric_relu_impl.hpp | 31 +++++++------------ .../tests/ann/layer/parametric_relu.cpp | 22 +++++++++---- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp index 679b39505d..240fe83aef 100644 --- a/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp +++ b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp @@ -22,28 +22,28 @@ namespace mlpack { template PReLUType::PReLUType(const double userAlpha) : - Layer() + Layer(), + userAlpha(userAlpha) { - alpha.set_size(1, 1); - alpha(0) = userAlpha; + // Nothing to do here. } template PReLUType::PReLUType( const PReLUType& other) : - Layer(other) + Layer(other), + userAlpha(other.userAlpha) { - userAlpha = other.userAlpha; - alpha = other.alpha; + // Nothing to do here. } template PReLUType::PReLUType( PReLUType&& other) : - Layer(std::move(other)) + Layer(std::move(other)), + userAlpha(std::move(other.userAlpha)) { - userAlpha = std::move(other.userAlpha); - alpha = std::move(other.alpha); + // Nothing to do here. } template @@ -54,7 +54,6 @@ PReLUType::operator=(const PReLUType& other) { Layer::operator=(other); userAlpha = other.userAlpha; - alpha = other.alpha; } return *this; @@ -68,7 +67,6 @@ PReLUType::operator=(PReLUType&& other) { Layer::operator=(std::move(other)); userAlpha = std::move(other.userAlpha); - alpha = std::move(other.alpha); } return *this; @@ -78,11 +76,7 @@ template void PReLUType::SetWeights( typename MatType::elem_type* weightsPtr) { - alpha = arma::mat(weightsPtr, 1, 1, false, false); - - //! Set value of alpha to the one given by user. - // TODO: this doesn't even make any sense. is it trainable or not? - // why is there userAlpha? is that for initialization only? + MakeAlias(alpha, weightsPtr, 1, 1); } template @@ -94,14 +88,14 @@ void PReLUType::CustomInitialize( throw std::invalid_argument("PReLUType::CustomInitialize(): wrong " "elements size!"); } - MakeAlias(alpha, W.memptr(), 1, 1); + + W(0) = userAlpha; } template void PReLUType::Forward( const MatType& input, MatType& output) { - // TODO: use transform()? output = input; if (this->training) { @@ -144,7 +138,6 @@ void PReLUType::serialize( ar(cereal::base_class>(this)); ar(CEREAL_NVP(userAlpha)); - ar(CEREAL_NVP(alpha)); } } // namespace mlpack diff --git a/src/mlpack/tests/ann/layer/parametric_relu.cpp b/src/mlpack/tests/ann/layer/parametric_relu.cpp index cc4680a398..1a9c610200 100644 --- a/src/mlpack/tests/ann/layer/parametric_relu.cpp +++ b/src/mlpack/tests/ann/layer/parametric_relu.cpp @@ -28,9 +28,13 @@ TEST_CASE("PReLUFORWARDTest", "[ANNLayerTest]") {-2.2, -1.5, 0.8}, {5.5, -4.7, 2.1}, {0.2, 0.1, -0.5}}; - PReLU model(0.01); + PReLU module(0.01); + module.Training() = true; + arma::mat moduleParams(module.WeightSize(), 1); + module.CustomInitialize(moduleParams, module.WeightSize()); + module.SetWeights((double*) moduleParams.memptr()); arma::mat predOutput; - model.Forward(input, predOutput); + module.Forward(input, predOutput); arma::mat actualOutput = {{0.5, 1.2, 3.1}, {-0.022, -0.015, 0.8}, {5.5, -0.047, 2.1}, @@ -48,13 +52,16 @@ TEST_CASE("PReLUBACKWARDTest", "[ANNLayerTest]") {-2.2, -1.5, 0.8}, {5.5, -4.7, 2.1}, {0.2, 0.1, -0.5}}; - PReLU model(0.01); + PReLU module(0.01); + arma::mat moduleParams(module.WeightSize(), 1); + module.CustomInitialize(moduleParams, module.WeightSize()); + module.SetWeights((double*) moduleParams.memptr()); arma::mat gy = {{0.2, -0.5, 0.8}, {1.5, -0.6, 0.1}, {-0.3, 0.2, -0.5}, {0.1, -0.1, 0.3}}; arma::mat predG; - model.Backward(input, gy, predG); + module.Backward(input, gy, predG); arma::mat actualG = {{0.2, -0.5, 0.8}, {0.015, -0.006, 0.1}, {-0.3, 0.002, -0.5}, @@ -73,13 +80,16 @@ TEST_CASE("PReLUGRADIENTTest", "[ANNLayerTest]") {-2.2, -1.5, 0.8}, {5.5, -4.7, 2.1}, {0.2, 0.1, -0.5}}; - PReLU model(0.01); + PReLU module(0.01); + arma::mat moduleParams(module.WeightSize(), 1); + module.CustomInitialize(moduleParams, module.WeightSize()); + module.SetWeights((double*) moduleParams.memptr()); arma::mat error = {{0.2, -0.5, 0.8}, {-0.015, -0.006, 0.001}, {-0.3, 0.002, -0.005}, {0.1, -0.1, 0.0035}}; arma::mat predGradient; - model.Gradient(input, error, predGradient); + module.Gradient(input, error, predGradient); REQUIRE(0.0103 - arma::accu(predGradient) == Approx(0.0).margin(1e-4)); From 9831d7ac972bf2bece33a3f878b1b31f3019cb6b Mon Sep 17 00:00:00 2001 From: Adarsh Santoria <108261986+AdarshSantoria@users.noreply.github.com> Date: Wed, 1 Mar 2023 22:55:52 +0530 Subject: [PATCH 84/88] Update src/mlpack/methods/ann/layer/parametric_relu_impl.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/ann/layer/parametric_relu_impl.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp index 240fe83aef..d0b9917e7b 100644 --- a/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp +++ b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp @@ -84,7 +84,8 @@ void PReLUType::CustomInitialize( MatType& W, const size_t elements) { - if (elements != 1) { + if (elements != 1) + { throw std::invalid_argument("PReLUType::CustomInitialize(): wrong " "elements size!"); } From 1e276557cb62845e0be338f5f1bee381c4fcc223 Mon Sep 17 00:00:00 2001 From: AdarshSantoria Date: Wed, 1 Mar 2023 23:44:50 +0530 Subject: [PATCH 85/88] remove similar tests --- .../not_adapted/activation_functions_test.cpp | 90 ------------------- 1 file changed, 90 deletions(-) diff --git a/src/mlpack/tests/ann/not_adapted/activation_functions_test.cpp b/src/mlpack/tests/ann/not_adapted/activation_functions_test.cpp index 4ba0505549..8a8b27b179 100644 --- a/src/mlpack/tests/ann/not_adapted/activation_functions_test.cpp +++ b/src/mlpack/tests/ann/not_adapted/activation_functions_test.cpp @@ -87,79 +87,6 @@ void CheckHardTanHDerivativeCorrect(const arma::colvec input, } }*/ -/** - * Implementation of the PReLU activation function test. The function - * is implemented as PReLU layer in the file parametric_relu.hpp. - * - * @param input Input data used for evaluating the PReLU activation - * function. - * @param target Target data used to evaluate the PReLU activation. - * -void CheckPReLUActivationCorrect(const arma::colvec input, - const arma::colvec target) -{ - PReLU<> prelu; - - // Test the activation function using the entire vector as input. - arma::colvec activations; - prelu.Forward(input, activations); - for (size_t i = 0; i < activations.n_elem; ++i) - { - REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Implementation of the PReLU activation function derivative test. - * The function is implemented as PReLU layer in the file - * parametric_relu.hpp - * - * @param input Input data used for evaluating the PReLU activation - * function. - * @param target Target data used to evaluate the PReLU activation. - * -void CheckPReLUDerivativeCorrect(const arma::colvec input, - const arma::colvec target) -{ - PReLU<> prelu; - - // Test the calculation of the derivatives using the entire vector as input. - arma::colvec derivatives; - - // This error vector will be set to 1 to get the derivatives. - arma::colvec error = arma::ones(input.n_elem); - prelu.Backward(input, error, derivatives); - for (size_t i = 0; i < derivatives.n_elem; ++i) - { - REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Implementation of the PReLU activation function gradient test. - * The function is implemented as PReLU layer in the file - * parametric_relu.hpp - * - * @param input Input data used for evaluating the PReLU activation - * function. - * @param target Target data used to evaluate the PReLU gradient. - * -void CheckPReLUGradientCorrect(const arma::colvec input, - const arma::colvec target) -{ - PReLU<> prelu; - - // Test the calculation of the derivatives using the entire vector as input. - arma::colvec gradient; - - // This error vector will be set to 1 to get the gradient. - arma::colvec error = arma::ones(input.n_elem); - prelu.Gradient(input, error, gradient); - REQUIRE(gradient.n_rows == 1); - REQUIRE(gradient.n_cols == 1); - REQUIRE(gradient(0) == Approx(target(0)).epsilon(1e-5)); -}*/ - /** * Implementation of the Hard Shrink activation function test. The function is * implemented as Hard Shrink layer in the file hardshrink.hpp @@ -460,23 +387,6 @@ TEST_CASE("HardTanHFunctionTest", "[ActivationFunctionsTest]") CheckHardTanHDerivativeCorrect(activationData, desiredDerivatives); }*/ -/** - * Basic test of the PReLU function. - * -TEST_CASE("PReLUFunctionTest", "[ActivationFunctionsTest]") -{ - const arma::colvec desiredActivations("-0.06 3.2 4.5 -3.006 \ - 1 -0.03 2 0"); - - const arma::colvec desiredDerivatives("0.03 1 1 0.03 \ - 1 0.03 1 1"); - const arma::colvec desiredGradient("-103.2"); - - CheckPReLUActivationCorrect(activationData, desiredActivations); - CheckPReLUDerivativeCorrect(desiredActivations, desiredDerivatives); - CheckPReLUGradientCorrect(activationData, desiredGradient); -}*/ - /** * Basic test of the CReLU function. * From 92301d52fdd2b91c71dd424cfdf9faa4c63aa496 Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Sun, 5 Mar 2023 09:01:13 -0600 Subject: [PATCH 86/88] Add CITATION file (and copy it in) (Closes: #3423) --- src/mlpack/bindings/R/CMakeLists.txt | 5 +++ src/mlpack/bindings/R/mlpack/inst/CITATION | 49 ++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 src/mlpack/bindings/R/mlpack/inst/CITATION diff --git a/src/mlpack/bindings/R/CMakeLists.txt b/src/mlpack/bindings/R/CMakeLists.txt index 873d01ca0c..170b4127a7 100644 --- a/src/mlpack/bindings/R/CMakeLists.txt +++ b/src/mlpack/bindings/R/CMakeLists.txt @@ -289,6 +289,11 @@ if (BUILD_R_BINDINGS) DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/mlpack/") + file(COPY + "${CMAKE_CURRENT_SOURCE_DIR}/mlpack/inst/CITATION" + DESTINATION + "${CMAKE_CURRENT_BINARY_DIR}/mlpack/inst") + # Do the actual build. add_custom_target(r_build ALL) diff --git a/src/mlpack/bindings/R/mlpack/inst/CITATION b/src/mlpack/bindings/R/mlpack/inst/CITATION new file mode 100644 index 0000000000..e925f71694 --- /dev/null +++ b/src/mlpack/bindings/R/mlpack/inst/CITATION @@ -0,0 +1,49 @@ +bibentry("Manual", + other = unlist(citation(auto = meta), recursive = FALSE)) + +bibentry("Article", + title = "mlpack 4: a fast, header-only C++ machine learning library", + author = c(person("Ryan R.", "Curtin", + comment = c(ORCID = "0000-0002-9903-8214")), + person("Marcus", "Edel", + comment = c(ORCID = "0000-0001-5445-7303")), + person("Omar", "Shrit", + comment = c(ORCID = "0000-0002-8621-3052")), + person("Shubham", "Agrawal", + comment = c(ORCID = "0000-0001-8713-4682")), + person("Suryoday", "Basak", + comment = c(ORCID = "0000-0002-1982-1787")), + person("James J.", "Balamuta", + comment = c(ORCID = "0000-0003-2826-8458")), + person("Ryan", "Birmingham", + comment = c(ORCID = "0000-0002-7943-6346")), + person("Kartik", "Dutt", + comment = c(ORCID = "0000-0003-3877-0142")), + person("Dirk", "Eddelbuettel", + comment = c(ORCID = "0000-0001-6419-907X")), + person("Rishabh", "Garg", + comment = c(ORCID = "0000-0003-0398-0887")), + person("Shikhar", "Jaiswal", + comment = c(ORCID = "0000-0002-3683-3931")), + person("Aakash", "Kaushik", + comment = c(ORCID = "0000-0003-1079-8338")), + person("Sangyeon", "Kim", + comment = c(ORCID = "0000-0003-0717-0240")), + person("Anjishnu", "Mukherjee", + comment = c(ORCID = "0000-0003-4012-8466")), + person("Nanubala Gnana", "Sai", + comment = c(ORCID = "0000-0003-0774-7994")), + person("Nippun", "Sharma", + comment = c(ORCID = "0000-0003-0365-2613")), + person("Yashwant Singh", "Parihar", + comment = c(ORCID = "0000-0003-3492-0377")), + person("Roshan", "Swain", + comment = c(ORCID = "0000-0002-7262-8230")), + person("Conrad", "Sanderson", + comment = c(ORCID = "0000-0002-0049-4501"))), + journal = "Journal of Open Source Software", + year = "2023", + volume = "8", + number = "82", + doi = "10.21105/joss.05026" + ) From 2ff0f57fb83b1cee5834594560c3fed12ebe4ffa Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 8 Mar 2023 09:14:25 -0500 Subject: [PATCH 87/88] Update HISTORY.md --- HISTORY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 14a6fade0a..a60d216931 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,7 +1,7 @@ ### mlpack ?.?.? ###### ????-??-?? - * Fix PReLU ann layer (#3420). + * Adapt PReLU layer for new neural network API (#3420). * Add CF decomposition methods: `QUIC_SVDPolicy` and `BlockKrylovSVDPolicy` (#3413, #3404). From 5453cb8032aa4774609be339e5c23383fde3d5b6 Mon Sep 17 00:00:00 2001 From: Aditya Raj <96882869+aadi-raj@users.noreply.github.com> Date: Sun, 19 Mar 2023 17:38:26 +0530 Subject: [PATCH 88/88] Adapt softmin layer (#3437) --- HISTORY.md | 2 + src/mlpack/methods/ann/layer/layer_types.hpp | 1 + .../ann/layer/not_adapted/softmin_impl.hpp | 56 ------------ .../methods/ann/layer/serialization.hpp | 1 + .../ann/layer/{not_adapted => }/softmin.hpp | 29 +++--- src/mlpack/methods/ann/layer/softmin_impl.hpp | 90 +++++++++++++++++++ src/mlpack/tests/ann/layer/softmin.cpp | 57 ++++++++++++ src/mlpack/tests/ann/layer_test.cpp | 1 + 8 files changed, 171 insertions(+), 66 deletions(-) delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/softmin_impl.hpp rename src/mlpack/methods/ann/layer/{not_adapted => }/softmin.hpp (72%) create mode 100644 src/mlpack/methods/ann/layer/softmin_impl.hpp create mode 100644 src/mlpack/tests/ann/layer/softmin.cpp diff --git a/HISTORY.md b/HISTORY.md index a60d216931..50f3f92efd 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,8 @@ ### mlpack ?.?.? ###### ????-??-?? + * Adapt Softmin layer for new neural network API (#3437). + * Adapt PReLU layer for new neural network API (#3420). * Add CF decomposition methods: `QUIC_SVDPolicy` and `BlockKrylovSVDPolicy` (#3413, #3404). diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 779cdc2716..dc12f47d7f 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -46,6 +46,7 @@ #include #include #include +#include // Convolution modes. #include diff --git a/src/mlpack/methods/ann/layer/not_adapted/softmin_impl.hpp b/src/mlpack/methods/ann/layer/not_adapted/softmin_impl.hpp deleted file mode 100644 index 6df396abcd..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/softmin_impl.hpp +++ /dev/null @@ -1,56 +0,0 @@ -/** - * @file methods/ann/layer/softmin_impl.hpp - * @author Aakash Kaushik - * - * Implementation of the Softmin class. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_SOFTMIN_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_SOFTMIN_IMPL_HPP - -// In case it hasn't yet been included. -#include "softmin.hpp" - -namespace mlpack { - -template -SoftminType::SoftminType() -{ - // Nothing to do here. -} - -template -void SoftminType::Forward( - const InputType& input, - OutputType& output) -{ - InputType softminInput = arma::exp(-(input.each_row() - - arma::min(input, 0))); - output = softminInput.each_row() / sum(softminInput, 0); -} - -template -void SoftminType::Backward( - const InputType& input, - const OutputType& gy, - OutputType& g) -{ - g = input % (gy - arma::repmat(arma::sum(gy % input), input.n_rows, 1)); -} - -template -template -void SoftminType::serialize( - Archive& ar, - const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); -} - -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/serialization.hpp b/src/mlpack/methods/ann/layer/serialization.hpp index 99cb7cc823..5819a5d604 100644 --- a/src/mlpack/methods/ann/layer/serialization.hpp +++ b/src/mlpack/methods/ann/layer/serialization.hpp @@ -68,6 +68,7 @@ CEREAL_REGISTER_TYPE(mlpack::PReLUType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::RBFType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::SoftmaxType<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::SoftminType<__VA_ARGS__>); \ CEREAL_REGISTER_MLPACK_LAYERS(arma::mat); diff --git a/src/mlpack/methods/ann/layer/not_adapted/softmin.hpp b/src/mlpack/methods/ann/layer/softmin.hpp similarity index 72% rename from src/mlpack/methods/ann/layer/not_adapted/softmin.hpp rename to src/mlpack/methods/ann/layer/softmin.hpp index 54b78d8e43..39cf1ee0d2 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/softmin.hpp +++ b/src/mlpack/methods/ann/layer/softmin.hpp @@ -24,14 +24,11 @@ namespace mlpack { * a vector of K real numbers, rescaling them so that the elements of the * K-dimensional output vector lie in the range [0, 1] and sum to 1. * - * @tparam InputType The type of the layer's inputs. The layer automatically - * cast inputs to this type (Default: arma::mat). - * @tparam OutputType The type of the computation which also causes the output - * to also be in this type. The type also allows the computation and weight - * type to differ from the input type (Default: arma::mat). + * @tparam MatType Matrix representation to accept as input and use for + * computation. */ -template -class SoftminType : public Layer +template +class SoftminType : public Layer { public: //! Create the Softmin object. @@ -40,6 +37,18 @@ class SoftminType : public Layer //! Clone the SoftminType object. This handles polymorphism correctly. SoftminType* Clone() const { return new SoftminType(*this); } + //! Virtual destructor. + virtual ~SoftminType() { } + + //! Copy the given SoftminType. + SoftminType(const SoftminType& other); + //! Take ownership of the given SoftminType. + SoftminType(SoftminType&& other); + //! Copy the given SoftminType. + SoftminType& operator=(const SoftminType& other); + //! Take ownership of the given SoftminType. + SoftminType& operator=(SoftminType&& other); + /** * Ordinary feed forward pass of a neural network, evaluating the function * f(x) by propagating the activity forward through f. @@ -47,7 +56,7 @@ class SoftminType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + void Forward(const MatType& input, MatType& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -58,7 +67,7 @@ class SoftminType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, const OutputType& gy, OutputType& g); + void Backward(const MatType& input, const MatType& gy, MatType& g); //! Serialize the layer. template @@ -68,7 +77,7 @@ class SoftminType : public Layer // Convenience typedefs. // Standard Softmin layer using no regularization. -typedef SoftminType Softmin; +typedef SoftminType Softmin; } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/softmin_impl.hpp b/src/mlpack/methods/ann/layer/softmin_impl.hpp new file mode 100644 index 0000000000..f668566f4a --- /dev/null +++ b/src/mlpack/methods/ann/layer/softmin_impl.hpp @@ -0,0 +1,90 @@ +/** + * @file methods/ann/layer/softmin_impl.hpp + * @author Aakash Kaushik + * + * Implementation of the Softmin class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_SOFTMIN_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_SOFTMIN_IMPL_HPP + +// In case it hasn't yet been included. +#include "softmin.hpp" + +namespace mlpack { + +template +SoftminType::SoftminType() +{ + // Nothing to do here. +} + +template +SoftminType::SoftminType(const SoftminType& other) : + Layer(other) +{ + // Nothing to do here. +} + +template +SoftminType::SoftminType(SoftminType&& other) : + Layer(std::move(other)) +{ + // Nothing to do here. +} + +template +SoftminType& +SoftminType::operator=(const SoftminType& other) +{ + if (this != &other) + Layer::operator=(other); + + return *this; +} + +template +SoftminType& +SoftminType::operator=(SoftminType&& other) +{ + if (this != &other) + Layer::operator=(std::move(other)); + + return *this; +} + +template +void SoftminType::Forward( + const MatType& input, + MatType& output) +{ + MatType softminInput = arma::exp(-(input.each_row() - + arma::min(input, 0))); + output = softminInput.each_row() / sum(softminInput, 0); +} + +template +void SoftminType::Backward( + const MatType& input, + const MatType& gy, + MatType& g) +{ + g = input % (gy - arma::repmat(arma::sum(gy % input), input.n_rows, 1)); +} + +template +template +void SoftminType::serialize( + Archive& ar, + const uint32_t /* version */) +{ + ar(cereal::base_class>(this)); +} + +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/ann/layer/softmin.cpp b/src/mlpack/tests/ann/layer/softmin.cpp new file mode 100644 index 0000000000..c11f446964 --- /dev/null +++ b/src/mlpack/tests/ann/layer/softmin.cpp @@ -0,0 +1,57 @@ +/** + * @file tests/ann/layer/softmin.cpp + * @author Aditya Raj + * + * Tests the ann layer modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include +#include + +#include "../../test_catch_tools.hpp" +#include "../../catch.hpp" +#include "../../serialization.hpp" +#include "../ann_test_tools.hpp" + +using namespace mlpack; +using namespace std; + +/** + * Simple Softmin module test. + */ +TEST_CASE("SimpleSoftminLayerTest", "[ANNLayerTest]") +{ + arma::mat input, output, gy, g; + Softmin module; + + // Test the forward function. + input = {{0.0, 0.1, 0.2}, + {1.0, 1.1, 1.2}, + {2.0, 2.1, 2.2}, + {2.9, 2.8, 2.5}}; + arma::mat actualOutput = {{0.641750, 0.636772, 0.623646}, + {0.236086, 0.234255, 0.229426}, + {0.086851, 0.086177, 0.084401}, + {0.035311, 0.042794, 0.062526}}; + + module.Forward(input, output); + REQUIRE(arma::accu(arma::abs(actualOutput - output)) == + Approx(0.0).margin(1e-4)); + + // Test the backward function. + gy = arma::zeros(input.n_rows, input.n_cols); + gy(1) = 1; + module.Backward(output, gy, g); + arma::mat calculatedGradient = {{-0.1515, 0, 0}, + {0.1803, 0, 0}, + {-0.0205, 0, 0}, + {-0.0083, 0, 0}}; + + REQUIRE(arma::accu(arma::abs(calculatedGradient - g)) == + Approx(0.0).margin(1e-04)); + +} diff --git a/src/mlpack/tests/ann/layer_test.cpp b/src/mlpack/tests/ann/layer_test.cpp index 77799e2c09..73456b23e9 100644 --- a/src/mlpack/tests/ann/layer_test.cpp +++ b/src/mlpack/tests/ann/layer_test.cpp @@ -35,3 +35,4 @@ #include "layer/padding.cpp" #include "layer/parametric_relu.cpp" #include "layer/softmax.cpp" +#include "layer/softmin.cpp"