From f85bf937d33bf2ef5bb1c189cd050ade31791aeb Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sat, 16 Apr 2022 21:26:01 -0400 Subject: [PATCH 01/39] First step: put the ANN tests in an ann/ directory. --- src/mlpack/tests/CMakeLists.txt | 52 +++++++++++-------- src/mlpack/tests/README.md | 13 +++-- .../{ => ann}/activation_functions_test.cpp | 2 +- src/mlpack/tests/{ => ann}/ann_dist_test.cpp | 4 +- src/mlpack/tests/{ => ann}/ann_layer_test.cpp | 6 +-- .../tests/{ => ann}/ann_regularizer_test.cpp | 4 +- src/mlpack/tests/{ => ann}/ann_test_tools.hpp | 0 .../{ => ann}/augmented_rnns_tasks_test.cpp | 2 +- .../tests/{ => ann}/convolution_test.cpp | 6 +-- .../{ => ann}/convolutional_network_test.cpp | 6 +-- src/mlpack/tests/{ => ann}/custom_layer.hpp | 0 .../{ => ann}/feedforward_network_2_test.cpp | 4 +- .../{ => ann}/feedforward_network_test.cpp | 5 +- .../tests/{ => ann}/init_rules_test.cpp | 2 +- src/mlpack/tests/{ => ann}/ksinit_test.cpp | 4 +- .../tests/{ => ann}/loss_functions_test.cpp | 6 +-- .../not_adapted}/async_learning_test.cpp | 0 .../{ => ann/not_adapted}/dcgan_test.cpp | 0 .../tests/{ => ann}/not_adapted/gan_test.cpp | 0 .../not_adapted/rbm_network_test.cpp | 0 .../not_adapted}/reward_clipping_test.cpp | 0 .../{ => ann/not_adapted}/rnn_reber_test.cpp | 0 .../tests/{ => ann}/not_adapted/wgan_test.cpp | 0 .../{ => ann}/recurrent_network_test.cpp | 4 +- .../tests/{ => ann}/rl_components_test.cpp | 4 +- 25 files changed, 68 insertions(+), 56 deletions(-) rename src/mlpack/tests/{ => ann}/activation_functions_test.cpp (99%) rename src/mlpack/tests/{ => ann}/ann_dist_test.cpp (99%) rename src/mlpack/tests/{ => ann}/ann_layer_test.cpp (99%) rename src/mlpack/tests/{ => ann}/ann_regularizer_test.cpp (97%) rename src/mlpack/tests/{ => ann}/ann_test_tools.hpp (100%) rename src/mlpack/tests/{ => ann}/augmented_rnns_tasks_test.cpp (99%) rename src/mlpack/tests/{ => ann}/convolution_test.cpp (99%) rename src/mlpack/tests/{ => ann}/convolutional_network_test.cpp (99%) rename src/mlpack/tests/{ => ann}/custom_layer.hpp (100%) rename src/mlpack/tests/{ => ann}/feedforward_network_2_test.cpp (98%) rename src/mlpack/tests/{ => ann}/feedforward_network_test.cpp (99%) rename src/mlpack/tests/{ => ann}/init_rules_test.cpp (99%) rename src/mlpack/tests/{ => ann}/ksinit_test.cpp (99%) rename src/mlpack/tests/{ => ann}/loss_functions_test.cpp (99%) rename src/mlpack/tests/{ => ann/not_adapted}/async_learning_test.cpp (100%) rename src/mlpack/tests/{ => ann/not_adapted}/dcgan_test.cpp (100%) rename src/mlpack/tests/{ => ann}/not_adapted/gan_test.cpp (100%) rename src/mlpack/tests/{ => ann}/not_adapted/rbm_network_test.cpp (100%) rename src/mlpack/tests/{ => ann/not_adapted}/reward_clipping_test.cpp (100%) rename src/mlpack/tests/{ => ann/not_adapted}/rnn_reber_test.cpp (100%) rename src/mlpack/tests/{ => ann}/not_adapted/wgan_test.cpp (100%) rename src/mlpack/tests/{ => ann}/recurrent_network_test.cpp (99%) rename src/mlpack/tests/{ => ann}/rl_components_test.cpp (99%) diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 6e6b0840b5..8f900961ab 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -3,18 +3,39 @@ include(CTest) # mlpack test executable. add_executable(mlpack_test EXCLUDE_FROM_ALL - activation_functions_test.cpp + # Tests for neural network code (and reinforcement learning code). + ann/activation_functions_test.cpp + ann/ann_dist_test.cpp + ann/ann_layer_test.cpp + ann/ann_regularizer_test.cpp + ann/ann_test_tools.hpp + ann/convolutional_network_test.cpp + ann/convolution_test.cpp + ann/custom_layer.hpp + ann/feedforward_network_test.cpp + ann/feedforward_network_2_test.cpp + ann/init_rules_test.cpp + ann/ksinit_test.cpp + ann/loss_functions_test.cpp + ann/recurrent_network_test.cpp + + ann/augmented_rnns_tasks_test.cpp + ann/rl_components_test.cpp + +# ann/not_adapted/reward_clipping_test.cpp +# ann/not_adapted/rbm_network_test.cpp +# ann/not_adapted/rnn_reber_test.cpp +# ann/not_adapted/async_learning_test.cpp +# ann/not_adapted/dcgan_test.cpp +# ann/not_adapted/gan_test.cpp +# ann/not_adapted/wgan_test.cpp + + # Tests for other methods. adaboost_test.cpp akfn_test.cpp aknn_test.cpp - ann_dist_test.cpp - ann_layer_test.cpp - ann_regularizer_test.cpp - ann_test_tools.hpp armadillo_svd_test.cpp arma_extend_test.cpp -# async_learning_test.cpp - augmented_rnns_tasks_test.cpp bayesian_linear_regression_test.cpp bias_svd_test.cpp binarize_test.cpp @@ -22,12 +43,9 @@ add_executable(mlpack_test callback_test.cpp cf_test.cpp cli_binding_test.cpp - convolutional_network_test.cpp - convolution_test.cpp cosine_tree_test.cpp cv_test.cpp dbscan_test.cpp -# dcgan_test.cpp decision_tree_regressor_test.cpp decision_tree_test.cpp det_test.cpp @@ -37,9 +55,6 @@ add_executable(mlpack_test emst_test.cpp facilities_test.cpp fastmks_test.cpp - feedforward_network_test.cpp - feedforward_network_2_test.cpp -# gan_test.cpp gmm_test.cpp hmm_test.cpp hpt_test.cpp @@ -47,7 +62,6 @@ add_executable(mlpack_test hyperplane_test.cpp image_load_test.cpp imputation_test.cpp - init_rules_test.cpp io_test.cpp kde_test.cpp kernel_pca_test.cpp @@ -57,7 +71,6 @@ add_executable(mlpack_test kmeans_test.cpp knn_test.cpp krann_search_test.cpp - ksinit_test.cpp lars_test.cpp lin_alg_test.cpp linear_regression_test.cpp @@ -67,7 +80,6 @@ add_executable(mlpack_test local_coordinate_coding_test.cpp logistic_regression_test.cpp log_test.cpp - loss_functions_test.cpp lsh_test.cpp main.cpp math_test.cpp @@ -94,13 +106,8 @@ add_executable(mlpack_test random_test.cpp randomized_svd_test.cpp range_search_test.cpp -# rbm_network_test.cpp rectangle_tree_test.cpp - recurrent_network_test.cpp -# rnn_reber_test.cpp regularized_svd_test.cpp -# reward_clipping_test.cpp - rl_components_test.cpp scaling_test.cpp size_checks_test.cpp serialization.cpp @@ -128,8 +135,9 @@ add_executable(mlpack_test ub_tree_test.cpp union_find_test.cpp vantage_point_tree_test.cpp -# wgan_test.cpp xgboost_test.cpp + + # Tests for individual bindings. main_tests/adaboost_test.cpp main_tests/adaboost_train_test.cpp main_tests/adaboost_classify_test.cpp diff --git a/src/mlpack/tests/README.md b/src/mlpack/tests/README.md index a8f9fee22d..8459313660 100644 --- a/src/mlpack/tests/README.md +++ b/src/mlpack/tests/README.md @@ -4,11 +4,13 @@ This directory contains code and data used to test all the algorithms and functi ## Test Directories Structure -- *_test.cpp - methods tests +- *_test.cpp - tests for non-neural-network methods +- ann/*_test.cpp - tests for code relating to neural networks (including + reinforcement learning) - main_tests/*_test.cpp - binding tests - data - data needed to run the tests -## Add tests +## Add tests We have a rich test suite, consisting of almost 2000 tests (and still counting). It is suggested to add tests when: @@ -17,7 +19,8 @@ We have a rich test suite, consisting of almost 2000 tests (and still counting). ## Building Tests -To build the test suite you can simply run `make mlpack_test`. +To build the test suite you can simply run `make mlpack_test` from a build +directory that has been properly configured with CMake.. ## To run Tests @@ -43,4 +46,6 @@ To run a single test, you can explicitly provide the name of the test, for examp `./bin/mlpack_test BinaryClassificationMetricsTest` -Catch2 provides many other features like filter, checkout the [Catch2 reference section](https://github.com/catchorg/Catch2/blob/devel/docs/Readme.md#top) - for more details. +Catch2 provides many other features like filtering; check out the +[Catch2 reference section](https://github.com/catchorg/Catch2/blob/devel/docs/Readme.md#top) +for more details. diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/ann/activation_functions_test.cpp similarity index 99% rename from src/mlpack/tests/activation_functions_test.cpp rename to src/mlpack/tests/ann/activation_functions_test.cpp index 2cadc4375c..26a87a075b 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/ann/activation_functions_test.cpp @@ -36,7 +36,7 @@ #include #include -#include "catch.hpp" +#include "../catch.hpp" using namespace mlpack; using namespace mlpack::ann; diff --git a/src/mlpack/tests/ann_dist_test.cpp b/src/mlpack/tests/ann/ann_dist_test.cpp similarity index 99% rename from src/mlpack/tests/ann_dist_test.cpp rename to src/mlpack/tests/ann/ann_dist_test.cpp index 4232c0042d..beb0e7387b 100644 --- a/src/mlpack/tests/ann_dist_test.cpp +++ b/src/mlpack/tests/ann/ann_dist_test.cpp @@ -16,8 +16,8 @@ #include #include -#include "catch.hpp" -#include "test_catch_tools.hpp" +#include "../catch.hpp" +#include "../test_catch_tools.hpp" #include diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann/ann_layer_test.cpp similarity index 99% rename from src/mlpack/tests/ann_layer_test.cpp rename to src/mlpack/tests/ann/ann_layer_test.cpp index cb1c5d766f..3f26afb4cf 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann/ann_layer_test.cpp @@ -22,10 +22,10 @@ #include #include -#include "test_catch_tools.hpp" -#include "catch.hpp" +#include "../test_catch_tools.hpp" +#include "../catch.hpp" +#include "../serialization.hpp" #include "ann_test_tools.hpp" -#include "serialization.hpp" using namespace mlpack; using namespace mlpack::ann; diff --git a/src/mlpack/tests/ann_regularizer_test.cpp b/src/mlpack/tests/ann/ann_regularizer_test.cpp similarity index 97% rename from src/mlpack/tests/ann_regularizer_test.cpp rename to src/mlpack/tests/ann/ann_regularizer_test.cpp index e300e6a9b1..a36818f196 100644 --- a/src/mlpack/tests/ann_regularizer_test.cpp +++ b/src/mlpack/tests/ann/ann_regularizer_test.cpp @@ -16,9 +16,9 @@ #include #include -#include "catch.hpp" +#include "../catch.hpp" +#include "../serialization.hpp" #include "ann_test_tools.hpp" -#include "serialization.hpp" using namespace mlpack; using namespace mlpack::ann; diff --git a/src/mlpack/tests/ann_test_tools.hpp b/src/mlpack/tests/ann/ann_test_tools.hpp similarity index 100% rename from src/mlpack/tests/ann_test_tools.hpp rename to src/mlpack/tests/ann/ann_test_tools.hpp diff --git a/src/mlpack/tests/augmented_rnns_tasks_test.cpp b/src/mlpack/tests/ann/augmented_rnns_tasks_test.cpp similarity index 99% rename from src/mlpack/tests/augmented_rnns_tasks_test.cpp rename to src/mlpack/tests/ann/augmented_rnns_tasks_test.cpp index 17fd801394..b0bcb24c5c 100644 --- a/src/mlpack/tests/augmented_rnns_tasks_test.cpp +++ b/src/mlpack/tests/ann/augmented_rnns_tasks_test.cpp @@ -22,7 +22,7 @@ #include #include -#include "catch.hpp" +#include "../catch.hpp" using std::vector; using std::pair; diff --git a/src/mlpack/tests/convolution_test.cpp b/src/mlpack/tests/ann/convolution_test.cpp similarity index 99% rename from src/mlpack/tests/convolution_test.cpp rename to src/mlpack/tests/ann/convolution_test.cpp index 1893804aa5..de85d810a9 100644 --- a/src/mlpack/tests/convolution_test.cpp +++ b/src/mlpack/tests/ann/convolution_test.cpp @@ -17,9 +17,9 @@ #include #include -#include "serialization.hpp" -#include "catch.hpp" -#include "test_catch_tools.hpp" +#include "../serialization.hpp" +#include "../catch.hpp" +#include "../test_catch_tools.hpp" using namespace mlpack; using namespace mlpack::ann; diff --git a/src/mlpack/tests/convolutional_network_test.cpp b/src/mlpack/tests/ann/convolutional_network_test.cpp similarity index 99% rename from src/mlpack/tests/convolutional_network_test.cpp rename to src/mlpack/tests/ann/convolutional_network_test.cpp index 91994e6b8c..9fc9c01694 100644 --- a/src/mlpack/tests/convolutional_network_test.cpp +++ b/src/mlpack/tests/ann/convolutional_network_test.cpp @@ -19,9 +19,9 @@ #include -#include "serialization.hpp" -#include "catch.hpp" -#include "test_catch_tools.hpp" +#include "../serialization.hpp" +#include "../catch.hpp" +#include "../test_catch_tools.hpp" using namespace mlpack; using namespace mlpack::ann; diff --git a/src/mlpack/tests/custom_layer.hpp b/src/mlpack/tests/ann/custom_layer.hpp similarity index 100% rename from src/mlpack/tests/custom_layer.hpp rename to src/mlpack/tests/ann/custom_layer.hpp diff --git a/src/mlpack/tests/feedforward_network_2_test.cpp b/src/mlpack/tests/ann/feedforward_network_2_test.cpp similarity index 98% rename from src/mlpack/tests/feedforward_network_2_test.cpp rename to src/mlpack/tests/ann/feedforward_network_2_test.cpp index a115468305..262ff83a39 100644 --- a/src/mlpack/tests/feedforward_network_2_test.cpp +++ b/src/mlpack/tests/ann/feedforward_network_2_test.cpp @@ -21,8 +21,8 @@ #include -#include "catch.hpp" -#include "serialization.hpp" +#include "../catch.hpp" +#include "../serialization.hpp" #include "custom_layer.hpp" using namespace mlpack; diff --git a/src/mlpack/tests/feedforward_network_test.cpp b/src/mlpack/tests/ann/feedforward_network_test.cpp similarity index 99% rename from src/mlpack/tests/feedforward_network_test.cpp rename to src/mlpack/tests/ann/feedforward_network_test.cpp index 613d0e56e8..a94a94020b 100644 --- a/src/mlpack/tests/feedforward_network_test.cpp +++ b/src/mlpack/tests/ann/feedforward_network_test.cpp @@ -18,9 +18,8 @@ #include -#include "catch.hpp" -#include "serialization.hpp" -//#include "custom_layer.hpp" +#include "../catch.hpp" +#include "../serialization.hpp" using namespace mlpack; using namespace mlpack::ann; diff --git a/src/mlpack/tests/init_rules_test.cpp b/src/mlpack/tests/ann/init_rules_test.cpp similarity index 99% rename from src/mlpack/tests/init_rules_test.cpp rename to src/mlpack/tests/ann/init_rules_test.cpp index 5d56e904b8..77514e60e5 100644 --- a/src/mlpack/tests/init_rules_test.cpp +++ b/src/mlpack/tests/ann/init_rules_test.cpp @@ -27,7 +27,7 @@ #include #include -#include "catch.hpp" +#include "../catch.hpp" using namespace mlpack; using namespace mlpack::ann; diff --git a/src/mlpack/tests/ksinit_test.cpp b/src/mlpack/tests/ann/ksinit_test.cpp similarity index 99% rename from src/mlpack/tests/ksinit_test.cpp rename to src/mlpack/tests/ann/ksinit_test.cpp index 15a9dec67c..0153e5b435 100644 --- a/src/mlpack/tests/ksinit_test.cpp +++ b/src/mlpack/tests/ann/ksinit_test.cpp @@ -10,8 +10,6 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#include "catch.hpp" - #include #include @@ -20,6 +18,8 @@ #include #include +#include "../catch.hpp" + using namespace mlpack; using namespace mlpack::ann; diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/ann/loss_functions_test.cpp similarity index 99% rename from src/mlpack/tests/loss_functions_test.cpp rename to src/mlpack/tests/ann/loss_functions_test.cpp index f7699b4fc9..daa9d67617 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/ann/loss_functions_test.cpp @@ -40,9 +40,9 @@ #include #include -#include "catch.hpp" -#include "test_catch_tools.hpp" -#include "ann_test_tools.hpp" +#include "../catch.hpp" +#include "../test_catch_tools.hpp" +#include "../ann_test_tools.hpp" using namespace mlpack; using namespace mlpack::ann; diff --git a/src/mlpack/tests/async_learning_test.cpp b/src/mlpack/tests/ann/not_adapted/async_learning_test.cpp similarity index 100% rename from src/mlpack/tests/async_learning_test.cpp rename to src/mlpack/tests/ann/not_adapted/async_learning_test.cpp diff --git a/src/mlpack/tests/dcgan_test.cpp b/src/mlpack/tests/ann/not_adapted/dcgan_test.cpp similarity index 100% rename from src/mlpack/tests/dcgan_test.cpp rename to src/mlpack/tests/ann/not_adapted/dcgan_test.cpp diff --git a/src/mlpack/tests/not_adapted/gan_test.cpp b/src/mlpack/tests/ann/not_adapted/gan_test.cpp similarity index 100% rename from src/mlpack/tests/not_adapted/gan_test.cpp rename to src/mlpack/tests/ann/not_adapted/gan_test.cpp diff --git a/src/mlpack/tests/not_adapted/rbm_network_test.cpp b/src/mlpack/tests/ann/not_adapted/rbm_network_test.cpp similarity index 100% rename from src/mlpack/tests/not_adapted/rbm_network_test.cpp rename to src/mlpack/tests/ann/not_adapted/rbm_network_test.cpp diff --git a/src/mlpack/tests/reward_clipping_test.cpp b/src/mlpack/tests/ann/not_adapted/reward_clipping_test.cpp similarity index 100% rename from src/mlpack/tests/reward_clipping_test.cpp rename to src/mlpack/tests/ann/not_adapted/reward_clipping_test.cpp diff --git a/src/mlpack/tests/rnn_reber_test.cpp b/src/mlpack/tests/ann/not_adapted/rnn_reber_test.cpp similarity index 100% rename from src/mlpack/tests/rnn_reber_test.cpp rename to src/mlpack/tests/ann/not_adapted/rnn_reber_test.cpp diff --git a/src/mlpack/tests/not_adapted/wgan_test.cpp b/src/mlpack/tests/ann/not_adapted/wgan_test.cpp similarity index 100% rename from src/mlpack/tests/not_adapted/wgan_test.cpp rename to src/mlpack/tests/ann/not_adapted/wgan_test.cpp diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/ann/recurrent_network_test.cpp similarity index 99% rename from src/mlpack/tests/recurrent_network_test.cpp rename to src/mlpack/tests/ann/recurrent_network_test.cpp index a7d3c459c3..1d12db8ee0 100644 --- a/src/mlpack/tests/recurrent_network_test.cpp +++ b/src/mlpack/tests/ann/recurrent_network_test.cpp @@ -19,8 +19,8 @@ #include -#include "catch.hpp" -#include "serialization.hpp" +#include "../catch.hpp" +#include "../serialization.hpp" using namespace mlpack; using namespace mlpack::ann; diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/ann/rl_components_test.cpp similarity index 99% rename from src/mlpack/tests/rl_components_test.cpp rename to src/mlpack/tests/ann/rl_components_test.cpp index 8ce209903e..4d1b7837cb 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/ann/rl_components_test.cpp @@ -23,8 +23,8 @@ #include #include -#include "catch.hpp" -#include "test_catch_tools.hpp" +#include "../catch.hpp" +#include "../test_catch_tools.hpp" using namespace mlpack; using namespace mlpack::rl; From 08062601878b41c0ed7aaf5d3a0b1f6ebc634b5c Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sat, 16 Apr 2022 21:35:43 -0400 Subject: [PATCH 02/39] Fix incorrect directory. --- src/mlpack/tests/ann/loss_functions_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/ann/loss_functions_test.cpp b/src/mlpack/tests/ann/loss_functions_test.cpp index daa9d67617..4e4fa51bff 100644 --- a/src/mlpack/tests/ann/loss_functions_test.cpp +++ b/src/mlpack/tests/ann/loss_functions_test.cpp @@ -42,7 +42,7 @@ #include "../catch.hpp" #include "../test_catch_tools.hpp" -#include "../ann_test_tools.hpp" +#include "ann_test_tools.hpp" using namespace mlpack; using namespace mlpack::ann; From b0db223118fcf71c46451544c5cfca10a2119b86 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sat, 16 Apr 2022 21:58:42 -0400 Subject: [PATCH 03/39] Split out tests that have not yet been re-enabled. --- src/mlpack/tests/CMakeLists.txt | 5 +- .../tests/ann/activation_functions_test.cpp | 775 --- src/mlpack/tests/ann/ann_layer_test.cpp | 5538 ---------------- .../tests/ann/feedforward_network_2_test.cpp | 143 - .../tests/ann/feedforward_network_test.cpp | 305 +- .../not_adapted/activation_functions_test.cpp | 817 +++ .../tests/ann/not_adapted/ann_layer_test.cpp | 5569 +++++++++++++++++ .../not_adapted/feedforward_network_test.cpp | 245 + .../not_adapted/recurrent_network_test.cpp | 464 ++ .../tests/ann/recurrent_network_test.cpp | 437 -- 10 files changed, 7184 insertions(+), 7114 deletions(-) delete mode 100644 src/mlpack/tests/ann/feedforward_network_2_test.cpp create mode 100644 src/mlpack/tests/ann/not_adapted/activation_functions_test.cpp create mode 100644 src/mlpack/tests/ann/not_adapted/ann_layer_test.cpp create mode 100644 src/mlpack/tests/ann/not_adapted/feedforward_network_test.cpp create mode 100644 src/mlpack/tests/ann/not_adapted/recurrent_network_test.cpp diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 8f900961ab..ff11ee9386 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -13,7 +13,6 @@ add_executable(mlpack_test ann/convolution_test.cpp ann/custom_layer.hpp ann/feedforward_network_test.cpp - ann/feedforward_network_2_test.cpp ann/init_rules_test.cpp ann/ksinit_test.cpp ann/loss_functions_test.cpp @@ -22,6 +21,10 @@ add_executable(mlpack_test ann/augmented_rnns_tasks_test.cpp ann/rl_components_test.cpp +# ann/not_adapted/activation_functions_test.cpp +# ann/not_adapted/ann_layer_test.cpp +# ann/not_adapted/feedforward_network_test.cpp +# ann/not_adapted/recurrent_network_test.cpp # ann/not_adapted/reward_clipping_test.cpp # ann/not_adapted/rbm_network_test.cpp # ann/not_adapted/rnn_reber_test.cpp diff --git a/src/mlpack/tests/ann/activation_functions_test.cpp b/src/mlpack/tests/ann/activation_functions_test.cpp index 26a87a075b..81a90e8bef 100644 --- a/src/mlpack/tests/ann/activation_functions_test.cpp +++ b/src/mlpack/tests/ann/activation_functions_test.cpp @@ -129,53 +129,6 @@ void CheckInverseCorrect(const arma::colvec input) } } -/** - * Implementation of the HardTanH activation function test. The function is - * implemented as a HardTanH Layer in hard_tanh.hpp - * - * @param input Input data used for evaluating the HardTanH activation function. - * @param target Target data used to evaluate the HardTanH activation. - * -void CheckHardTanHActivationCorrect(const arma::colvec input, - const arma::colvec target) -{ - HardTanH<> htf; - - // Test the activation function using the entire vector as input. - arma::colvec activations; - htf.Forward(input, activations); - for (size_t i = 0; i < activations.n_elem; ++i) - { - REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Implementation of the HardTanH activation function derivative test. The - * derivative is implemented as HardTanH Layer in hard_tanh.hpp - * - * @param input Input data used for evaluating the HardTanH activation - * function. - * @param target Target data used to evaluate the HardTanH activation. - * -void CheckHardTanHDerivativeCorrect(const arma::colvec input, - const arma::colvec target) -{ - HardTanH<> htf; - - // Test the calculation of the derivatives using the entire vector as input. - arma::colvec derivatives; - - // This error vector will be set to 1 to get the derivatives. - arma::colvec error = arma::ones(input.n_elem); - htf.Backward(input, error, derivatives); - - for (size_t i = 0; i < derivatives.n_elem; ++i) - { - REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - /** * Implementation of the LeakyReLU activation function test. The function is * implemented as LeakyReLU layer in the file leaky_relu.hpp @@ -224,533 +177,6 @@ void CheckLeakyReLUDerivativeCorrect(const arma::colvec input, } } -/** - * Implementation of the ELU activation function test. The function is - * implemented as ELU layer in the file elu.hpp - * - * @param input Input data used for evaluating the ELU activation function. - * @param target Target data used to evaluate the ELU activation. - * -void CheckELUActivationCorrect(const arma::colvec input, - const arma::colvec target) -{ - // Initialize ELU object with alpha = 1.0. - ELU<> lrf(1.0); - - // Test the activation function using the entire vector as input. - arma::colvec activations; - lrf.Forward(input, activations); - for (size_t i = 0; i < activations.n_elem; ++i) - { - REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Implementation of the ELU activation function derivative test. The function - * is implemented as ELU layer in the file elu.hpp - * - * @param input Input data used for evaluating the ELU activation function. - * @param target Target data used to evaluate the ELU activation. - * -void CheckELUDerivativeCorrect(const arma::colvec input, - const arma::colvec target) -{ - // Initialize ELU object with alpha = 1.0. - ELU<> lrf(1.0); - - // Test the calculation of the derivatives using the entire vector as input. - arma::colvec derivatives, activations; - - // This error vector will be set to 1 to get the derivatives. - arma::colvec error = arma::ones(input.n_elem); - lrf.Forward(input, activations); - lrf.Backward(activations, error, derivatives); - for (size_t i = 0; i < derivatives.n_elem; ++i) - { - REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Implementation of the PReLU activation function test. The function - * is implemented as PReLU layer in the file parametric_relu.hpp. - * - * @param input Input data used for evaluating the PReLU activation - * function. - * @param target Target data used to evaluate the PReLU activation. - * -void CheckPReLUActivationCorrect(const arma::colvec input, - const arma::colvec target) -{ - PReLU<> prelu; - - // Test the activation function using the entire vector as input. - arma::colvec activations; - prelu.Forward(input, activations); - for (size_t i = 0; i < activations.n_elem; ++i) - { - REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Implementation of the PReLU activation function derivative test. - * The function is implemented as PReLU layer in the file - * parametric_relu.hpp - * - * @param input Input data used for evaluating the PReLU activation - * function. - * @param target Target data used to evaluate the PReLU activation. - * -void CheckPReLUDerivativeCorrect(const arma::colvec input, - const arma::colvec target) -{ - PReLU<> prelu; - - // Test the calculation of the derivatives using the entire vector as input. - arma::colvec derivatives; - - // This error vector will be set to 1 to get the derivatives. - arma::colvec error = arma::ones(input.n_elem); - prelu.Backward(input, error, derivatives); - for (size_t i = 0; i < derivatives.n_elem; ++i) - { - REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Implementation of the PReLU activation function gradient test. - * The function is implemented as PReLU layer in the file - * parametric_relu.hpp - * - * @param input Input data used for evaluating the PReLU activation - * function. - * @param target Target data used to evaluate the PReLU gradient. - * -void CheckPReLUGradientCorrect(const arma::colvec input, - const arma::colvec target) -{ - PReLU<> prelu; - - // Test the calculation of the derivatives using the entire vector as input. - arma::colvec gradient; - - // This error vector will be set to 1 to get the gradient. - arma::colvec error = arma::ones(input.n_elem); - prelu.Gradient(input, error, gradient); - REQUIRE(gradient.n_rows == 1); - REQUIRE(gradient.n_cols == 1); - REQUIRE(gradient(0) == Approx(target(0)).epsilon(1e-5)); -}*/ - -/** - * Implementation of the Hard Shrink activation function test. The function is - * implemented as Hard Shrink layer in the file hardshrink.hpp - * - * @param input Input data used for evaluating the Hard Shrink activation function. - * @param target Target data used to evaluate the Hard Shrink activation. - * -void CheckHardShrinkActivationCorrect(const arma::colvec input, - const arma::colvec target) -{ - HardShrink<> hardshrink; - - // Test the activation function using the entire vector as input. - arma::colvec activations; - hardshrink.Forward(input, activations); - for (size_t i = 0; i < activations.n_elem; ++i) - { - REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Implementation of the HardShrink activation function derivative test. - * The derivative function is implemented as HardShrink layer in the file - * hardshrink.hpp - * - * @param input Input data used for evaluating the HardShrink activation - * function. - * @param target Target data used to evaluate the HardShrink activation. - * -void CheckHardShrinkDerivativeCorrect(const arma::colvec input, - const arma::colvec target) -{ - HardShrink<> hardshrink; - - // Test the calculation of the derivatives using the entire vector as input. - arma::colvec derivatives; - - // This error vector will be set to 1 to get the derivatives. - arma::colvec error = arma::ones(input.n_elem); - hardshrink.Backward(input, error, derivatives); - for (size_t i = 0; i < derivatives.n_elem; ++i) - { - REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Implementation of the Soft Shrink activation function test. The function is - * implemented as Soft Shrink layer in the file softshrink.hpp. - * - * @param input Input data used for evaluating the Soft Shrink activation - * function. - * @param target Target data used to evaluate the Soft Shrink activation. - * -void CheckSoftShrinkActivationCorrect(const arma::colvec input, - const arma::colvec target) -{ - SoftShrink<> softshrink; - - // Test the activation function using the entire vector as input. - arma::colvec activations; - softshrink.Forward(input, activations); - for (size_t i = 0; i < activations.n_elem; ++i) - { - REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Implementation of the Soft Shrink activation function derivative test. - * The derivative function is implemented as Soft Shrink layer in the file - * softshrink.hpp - * - * @param input Input data used for evaluating the Soft Shrink activation - * function. - * @param target Target data used to evaluate the Soft Shrink activation. - * -void CheckSoftShrinkDerivativeCorrect(const arma::colvec input, - const arma::colvec target) -{ - SoftShrink<> softshrink; - - // Test the calculation of the derivatives using the entire vector as input. - arma::colvec derivatives; - - // This error vector will be set to 1 to get the derivatives. - arma::colvec error = arma::ones(input.n_elem); - softshrink.Backward(input, error, derivatives); - for (size_t i = 0; i < derivatives.n_elem; ++i) - { - REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Simple SELU activation test to check whether the mean and variance remain - * invariant after passing normalized inputs through the function. - * -TEST_CASE("SELUFunctionNormalizedTest", "[ActivationFunctionsTest]") -{ - arma::mat input = arma::randn(1000, 1); - - arma::mat output; - - SELU selu; - - selu.Forward(input, output); - - REQUIRE(arma::as_scalar(arma::abs(arma::mean(input) - - arma::mean(output))) <= 0.1); - - REQUIRE(arma::as_scalar(arma::abs(arma::var(input) - - arma::var(output))) <= 0.1); -}*/ - -/** - * Simple SELU activation test to check whether the mean and variance - * vary significantly after passing unnormalized inputs through the function. - * -TEST_CASE("SELUFunctionUnnormalizedTest", "[ActivationFunctionsTest]") -{ - const arma::colvec input("5.96402758 0.9966824 0.99975321 1 \ - 7.76159416 -0.76159416 0.96402758 8"); - - arma::mat output; - - SELU selu; - - selu.Forward(input, output); - - REQUIRE(arma::as_scalar(arma::abs(arma::mean(input) - - arma::mean(output))) >= 0.1); - - REQUIRE(arma::as_scalar(arma::abs(arma::var(input) - - arma::var(output))) >= 0.1); -}*/ - -/** - * Simple SELU derivative test to check whether the derivatives - * produced by the activation function are correct. - * - * -TEST_CASE("SELUFunctionDerivativeTest", "[ActivationFunctionsTest]") -{ - arma::mat input = arma::ones(1000, 1); - - arma::mat error = arma::ones(input.n_elem, 1); - - arma::mat derivatives, activations; - - SELU selu; - - selu.Forward(input, activations); - selu.Backward(activations, error, derivatives); - - REQUIRE(arma::as_scalar(arma::abs(arma::mean(derivatives) - - selu.Lambda())) <= 10e-4); - - input.fill(-1); - - selu.Forward(input, activations); - selu.Backward(activations, error, derivatives); - - REQUIRE(arma::as_scalar(arma::abs(arma::mean(derivatives) - - selu.Lambda() * selu.Alpha() - arma::mean(activations))) <= 10e-4); -}*/ - -/** - * Implementation of the CELU activation function test. The function is - * implemented as CELU layer in the file celu.hpp. - * - * @param input Input data used for evaluating the CELU activation function. - * @param target Target data used to evaluate the CELU activation. - * -void CheckCELUActivationCorrect(const arma::colvec input, - const arma::colvec target) -{ - // Initialize CELU object with alpha = 1.0. - CELU<> lrf(1.0); - - // Test the activation function using the entire vector as input. - arma::colvec activations; - lrf.Forward(input, activations); - for (size_t i = 0; i < activations.n_elem; ++i) - { - REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Implementation of the CELU activation function derivative test. The function - * is implemented as CELU layer in the file celu.hpp. - * - * @param input Input data used for evaluating the CELU activation function. - * @param target Target data used to evaluate the CELU activation. - * -void CheckCELUDerivativeCorrect(const arma::colvec input, - const arma::colvec target) -{ - // Initialize CELU object with alpha = 1.0. - CELU<> lrf(1.0); - - // Test the calculation of the derivatives using the entire vector as input. - arma::colvec derivatives, activations; - - // This error vector will be set to 1 to get the derivatives. - arma::colvec error = arma::ones(input.n_elem); - lrf.Forward(input, activations); - lrf.Backward(activations, error, derivatives); - for (size_t i = 0; i < derivatives.n_elem; ++i) - { - REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Implementation of the ISRLU activation function test. The function is - * implemented as ISRLU layer in the file isrlu.hpp. - * - * @param input Input data used for evaluating the ISRLU activation function. - * @param target Target data used to evaluate the ISRLU activation. - * -void CheckISRLUActivationCorrect(const arma::colvec input, - const arma::colvec target) -{ - // Initialize ISRLU object with alpha = 1.0. - ISRLU<> lrf(1.0); - - // Test the activation function using the entire vector as input. - arma::colvec activations; - lrf.Forward(input, activations); - for (size_t i = 0; i < activations.n_elem; ++i) - { - REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Implementation of the ISRLU activation function derivative test. The function - * is implemented as ISRLU layer in the file isrlu.hpp. - * - * @param input Input data used for evaluating the ISRLU activation function. - * @param target Target data used to evaluate the ISRLU activation. - * -void CheckISRLUDerivativeCorrect(const arma::colvec input, - const arma::colvec target) -{ - // Initialize ISRLU object with alpha = 1.0. - ISRLU<> lrf(1.0); - - // Test the calculation of the derivatives using the entire vector as input. - arma::colvec derivatives, activations; - - // This error vector will be set to 1 to get the derivatives. - arma::colvec error = arma::ones(input.n_elem); - lrf.Forward(input, activations); - lrf.Backward(activations, error, derivatives); - for (size_t i = 0; i < derivatives.n_elem; ++i) - { - REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Implementation of the Softmin activation function test. The function is - * implemented as Softmin layer in the file softmin.hpp. - * - * @param input Input data used for evaluating the Softmin activation function. - * @param target Target data used to evaluate the Softmin activation. - * -void CheckSoftminActivationCorrect(const arma::colvec input, - const arma::colvec target) -{ - // Initialize Softmin object. - Softmin<> softmin; - - // Test the activation function using the entire vector as input. - arma::colvec activations; - softmin.Forward(input, activations); - for (size_t i = 0; i < activations.n_elem; ++i) - { - REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Implementation of the Softmin activation function derivative test. - * The function is implemented as Softmin layer in the file softmin.hpp. - * - * @param input Input data used for evaluating the Softmin activation function. - * @param target Target data used to evaluate the Softmin activation. - * -void CheckSoftminDerivativeCorrect(const arma::colvec input, - const arma::colvec target) -{ - // Initialize Softmin object. - Softmin<> softmin; - - // Test the calculation of the derivatives using the entire vector as input. - arma::colvec derivatives, activations; - - // This error vector will be set to [[1.0],[0.0],[1.0],[0.0]] - // to get the derivatives. - arma::colvec error = arma::ones(input.n_elem); - error(1) = 0.0; - error(3) = 0.0; - softmin.Forward(input, activations); - softmin.Backward(activations, error, derivatives); - for (size_t i = 0; i < derivatives.n_elem; ++i) - { - REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Implementation of the Flatten T Swish activation function test. The function is - * implemented as Flatten T Swish layer in the file flatten_t_swish.hpp. - * - * @param input Input data used for evaluating the Flatten T Swish activation - * function. - * @param target Target data used to evaluate the Flatten T Swish activation. - * -void CheckFlattenTSwishActivationCorrect(const arma::colvec input, - const arma::colvec target) -{ - FlattenTSwish<> fts(0.4); - arma::colvec activations; - - fts.Forward(input, activations); - for (size_t i = 0; i < activations.n_elem; ++i) - { - REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Implementation of the Softmin activation function derivative test. - * The function is implemented as Softmin layer in the file softmin.hpp. - * - * @param input Input data used for evaluating the Softmin activation function. - * @param target Target data used to evaluate the Softmin activation. - * -void CheckFlattenTSwishDerivateCorrect(const arma::colvec input, - const arma::colvec target) -{ - FlattenTSwish<> fts; - - // Set the error to 1 to get the actual derivative. - arma::colvec error = arma::ones(input.n_elem); - - arma::colvec derivate; - fts.Backward(input, error, derivate); - for (size_t i = 0; i < derivate.n_elem; ++i) - { - REQUIRE(derivate.at(i) == Approx(target.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Implementation of the ReLU6 activation function derivative test. The function - * is implemented as ReLU6 layer in the file relu6.hpp. - * - * @param input Input data used for evaluating the ReLU6 activation function. - * @param target Target data used to evaluate the ReLU6 activation. - * -void CheckReLU6Correct(const arma::colvec input, - const arma::colvec ActivationTarget, - const arma::colvec DerivativeTarget) -{ - // Initialize ReLU6 object. - ReLU6<> relu6; - - // Test the calculation of the derivatives using the entire vector as input. - arma::colvec derivatives, activations; - - // This error vector will be set to 1 to get the derivatives. - arma::colvec error = arma::ones(input.n_elem); - relu6.Forward(input, activations); - for (size_t i = 0; i < activations.n_elem; ++i) - { - REQUIRE(activations.at(i) == Approx(ActivationTarget.at(i)).epsilon(1e-5)); - } - relu6.Backward(activations, error, derivatives); - for (size_t i = 0; i < derivatives.n_elem; ++i) - { - REQUIRE(derivatives.at(i) == Approx(DerivativeTarget.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Basic test of the ReLU6 function. - * -TEST_CASE("ReLU6FunctionTest", "[ActivationFunctionsTest]") -{ - const arma::colvec activationData("-2.0 3.0 0.0 6.0 24.0"); - - // desiredActivations taken from PyTorch. - const arma::colvec desiredActivations("0.0 3.0 0.0 6.0 6.0"); - - // desiredDerivatives taken from PyTorch. - const arma::colvec desiredDerivatives("0.0 1.0 0.0 0.0 0.0"); - - CheckReLU6Correct(activationData, desiredActivations, desiredDerivatives); -}*/ - /** * Basic test of the tanh function. */ @@ -845,36 +271,6 @@ TEST_CASE("LeakyReLUFunctionTest", "[ActivationFunctionsTest]") CheckLeakyReLUDerivativeCorrect(desiredActivations, desiredDerivatives); } -/** - * Basic test of the HardTanH function. - * -TEST_CASE("HardTanHFunctionTest", "[ActivationFunctionsTest]") -{ - const arma::colvec desiredActivations("-1 1 1 -1 \ - 1 -1 1 0"); - - const arma::colvec desiredDerivatives("0 0 0 0 \ - 1 1 0 1"); - - CheckHardTanHActivationCorrect(activationData, desiredActivations); - CheckHardTanHDerivativeCorrect(activationData, desiredDerivatives); -}*/ - -/** - * Basic test of the ELU function. - * -TEST_CASE("ELUFunctionTest", "[ActivationFunctionsTest]") -{ - const arma::colvec desiredActivations("-0.86466471 3.2 4.5 -1.0 \ - 1 -0.63212055 2 0"); - - const arma::colvec desiredDerivatives("0.13533529 1 1 0 \ - 1 0.36787945 1 1"); - - CheckELUActivationCorrect(activationData, desiredActivations); - CheckELUDerivativeCorrect(activationData, desiredDerivatives); -}*/ - /** * Basic test of the softplus function. */ @@ -896,73 +292,6 @@ TEST_CASE("SoftplusFunctionTest", "[ActivationFunctionsTest]") CheckInverseCorrect(desiredActivations); } -/** - * Basic test of the PReLU function. - * -TEST_CASE("PReLUFunctionTest", "[ActivationFunctionsTest]") -{ - const arma::colvec desiredActivations("-0.06 3.2 4.5 -3.006 \ - 1 -0.03 2 0"); - - const arma::colvec desiredDerivatives("0.03 1 1 0.03 \ - 1 0.03 1 1"); - const arma::colvec desiredGradient("-103.2"); - - CheckPReLUActivationCorrect(activationData, desiredActivations); - CheckPReLUDerivativeCorrect(desiredActivations, desiredDerivatives); - CheckPReLUGradientCorrect(activationData, desiredGradient); -}*/ - -/** - * Basic test of the CReLU function. - * -TEST_CASE("CReLUFunctionTest", "[ActivationFunctionsTest]") -{ - const arma::colvec desiredActivations("0 3.2 4.5 0 \ - 1 0 2 0 2 0 0 \ - 100.2 0 1 0 0"); - - const arma::colvec desiredDerivatives("0 0 0 0 \ - 0 0 0 0"); - CReLU<> crelu; - // Test the activation function using the entire vector as input. - arma::colvec activations; - crelu.Forward(activationData, activations); - arma::colvec derivatives; - // This error vector will be set to 1 to get the derivatives. - arma::colvec error = arma::ones(desiredActivations.n_elem); - crelu.Backward(desiredActivations, error, derivatives); - for (size_t i = 0; i < activations.n_elem; ++i) - { - REQUIRE(activations.at(i) == - Approx(desiredActivations.at(i)).epsilon(1e-5)); - } - for (size_t i = 0; i < derivatives.n_elem; ++i) - { - REQUIRE(derivatives.at(i) == - Approx(desiredDerivatives.at(i)).epsilon(1e-5)); - } -}*/ - -/** - * Basic test of the swish function. - * -TEST_CASE("SwishFunctionTest", "[ActivationFunctionsTest]") -{ - // Hand-calculated values using Python interpreter. - const arma::colvec desiredActivations("-0.238405 3.07466 4.45055 \ - -3.05183208657e-42 0.731058 -0.26894 \ - 1.76159 0"); - - const arma::colvec desiredDerivatives("0.3819171 1.0856295 1.039218 \ - 0.5 0.83540367 0.3671335 1.073787\ - 0.5"); - - CheckActivationCorrect(activationData, desiredActivations); - CheckDerivativeCorrect(desiredActivations, - desiredDerivatives); -}*/ - /** * Basic test of the hard sigmoid function. */ @@ -1047,21 +376,6 @@ TEST_CASE("GELUFunctionTest", "[ActivationFunctionsTest]") desiredDerivatives); } -/** - * Basic test of the Hard Shrink function. - * -TEST_CASE("HardShrinkFunctionTest", "[ActivationFunctionsTest]") -{ - const arma::colvec desiredActivations("-2 3.2 4.5 -100.2 1 -1 2 0"); - - const arma::colvec desiredDerivatives("1 1 1 1 1 1 1 0"); - - CheckHardShrinkActivationCorrect(activationData, - desiredActivations); - CheckHardShrinkDerivativeCorrect(desiredActivations, - desiredDerivatives); -}*/ - /** * Basic test of the Elliot function. */ @@ -1102,53 +416,6 @@ TEST_CASE("ElishFunctionTest", "[ActivationFunctionsTest]") desiredDerivatives); } -/** - * Basic test of the Soft Shrink function. - * -TEST_CASE("SoftShrinkFunctionTest", "[ActivationFunctionsTest]") -{ - const arma::colvec desiredActivations("-1.5 2.7 4 -99.7 0.5 -0.5 1.5 0"); - - const arma::colvec desiredDerivatives("1 1 1 1 1 1 1 0"); - - CheckSoftShrinkActivationCorrect(activationData, - desiredActivations); - CheckSoftShrinkDerivativeCorrect(desiredActivations, - desiredDerivatives); -}*/ - -/** - * Basic test of the CELU activation function. - * -TEST_CASE("CELUFunctionTest", "[ActivationFunctionsTest]") -{ - const arma::colvec desiredActivations("-0.86466472 3.2 4.5 \ - -1 1 -0.63212056 2 0"); - - const arma::colvec desiredDerivatives("0.42119275 1 1 \ - 0.36787944 1 \ - 0.5314636 1 1"); - - CheckCELUActivationCorrect(activationData, desiredActivations); - CheckCELUDerivativeCorrect(desiredActivations, desiredDerivatives); -}*/ - -/** - * Basic test of the ISRLU activation function. - * -TEST_CASE("ISRLUFunctionTest", "[ActivationFunctionsTest]") -{ - const arma::colvec desiredActivations("-0.89442719 3.2 4.5 \ - -0.99995020 1 -0.70710678 2 0"); - - const arma::colvec desiredDerivatives("0.41408666 1 1 \ - 0.35357980 1 \ - 0.54433105 1 1"); - - CheckISRLUActivationCorrect(activationData, desiredActivations); - CheckISRLUDerivativeCorrect(activationData, desiredDerivatives); -}*/ - /** * Basic test of the inverse quadratic function. */ @@ -1273,26 +540,6 @@ TEST_CASE("GaussianFunctionTest", "[ActivationFunctionsTest]") desiredDerivatives); } -/** - * Basic test of the Softmin function. - * -TEST_CASE("SoftminFunctionTest", "[ActivationFunctionsTest]") -{ - const arma::colvec activationData("4.2 2.4 7.0 6.4"); - - // Hand-calculated Values. - const arma::colvec desiredActivations("0.1384799751 0.8377550303 \ - 0.008420976 0.0153440186"); - - const arma::colvec desiredDerivatives("0.1181371351 -0.12306701070 \ - 0.0071839266 -0.0022540509"); - - CheckSoftminActivationCorrect(activationData, - desiredActivations); - CheckSoftminDerivativeCorrect(activationData, - desiredDerivatives); -}*/ - /** * Basic test of the Hard Swish function. */ @@ -1357,25 +604,3 @@ TEST_CASE("SILUFunctionTest", "[ActivationFunctionsTest]") CheckActivationCorrect(activationData, desiredActivation); CheckDerivativeCorrect(desiredActivation, desiredDerivate); } - -/** - * Basic test of Flatten T Swish function. - * -TEST_CASE("FlattenTSwishFunctionTest", "[ActivationFunctionsTest]") -{ - // Random Value. - arma::colvec input("-4.0 -1.0 2 3 4 5 6"); - - // Hand Calculated and using PyTorch. - arma::colvec desiredActivation( - "0.4000000059604645 0.4000000059604645 2.1615941524505615 \ - 3.2577223777770996 4.328054904937744 5.3665361404418945 \ - 6.385164737701416"); - - // Hand Calculated and using PyTorch. - arma::colvec desiredDerivation("0.694792 0.694792 1.096893 1.079178 1.042602 \ - 1.020182 1.009048"); - - CheckFlattenTSwishActivationCorrect(input, desiredActivation); - CheckFlattenTSwishDerivateCorrect(desiredActivation, desiredDerivation); -}*/ diff --git a/src/mlpack/tests/ann/ann_layer_test.cpp b/src/mlpack/tests/ann/ann_layer_test.cpp index 3f26afb4cf..8e911501e2 100644 --- a/src/mlpack/tests/ann/ann_layer_test.cpp +++ b/src/mlpack/tests/ann/ann_layer_test.cpp @@ -30,214 +30,6 @@ using namespace mlpack; using namespace mlpack::ann; -// // network1 should be allocated with `new`, and trained on some data. -// template -// void CheckRNNCopyFunction(ModelType* network1, -// MatType& trainData, -// MatType& trainLabels, -// const size_t maxEpochs) -// { -// arma::cube predictions1; -// arma::cube predictions2; -// ens::StandardSGD opt(0.1, 1, maxEpochs * trainData.n_slices, -100, false); - -// network1->Train(trainData, trainLabels, opt); -// network1->Predict(trainData, predictions1); - -// RNN<> network2 = *network1; -// delete network1; - -// // Deallocating all of network1's memory, so that network2 does not use any -// // of that memory. -// network2.Predict(trainData, predictions2); -// CheckMatrices(predictions1, predictions2); -// } - -// // network1 should be allocated with `new`, and trained on some data. -// template -// void CheckRNNMoveFunction(ModelType* network1, -// MatType& trainData, -// MatType& trainLabels, -// const size_t maxEpochs) -// { -// arma::cube predictions1; -// arma::cube predictions2; -// ens::StandardSGD opt(0.1, 1, maxEpochs * trainData.n_slices, -100, false); - -// network1->Train(trainData, trainLabels, opt); -// network1->Predict(trainData, predictions1); - -// RNN<> network2(std::move(*network1)); -// delete network1; - -// // Deallocating all of network1's memory, so that network2 does not use any -// // of that memory. -// network2.Predict(trainData, predictions2); -// CheckMatrices(predictions1, predictions2); -// } - -/** - * Simple add module test. - * -TEST_CASE("SimpleAddLayerTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta; - Add module(10); - module.Parameters().randu(); - - // Test the Forward function. - input = arma::zeros(10, 1); - module.Forward(input, output); - REQUIRE(arma::accu(module.Parameters()) == arma::accu(output)); - - // Test the Backward function. - module.Backward(input, output, delta); - REQUIRE(arma::accu(output) == arma::accu(delta)); - - // Test the forward function. - input = arma::ones(10, 1); - module.Forward(input, output); - REQUIRE(10 + arma::accu(module.Parameters()) == - Approx(arma::accu(output)).epsilon(1e-5)); - - // Test the backward function. - module.Backward(input, output, delta); - REQUIRE(arma::accu(output) == Approx(arma::accu(delta)).epsilon(1e-5)); -} -*/ - -/** - * Jacobian add module test. - * -TEST_CASE("JacobianAddLayerTest", "[ANNLayerTest]") -{ - for (size_t i = 0; i < 5; ++i) - { - const size_t elements = math::RandInt(2, 1000); - arma::mat input; - input.set_size(elements, 1); - - Add module(elements); - module.Parameters().randu(); - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -} -*/ - -/** - * Add layer numerical gradient test. - * -TEST_CASE("GradientAddLayerTest", "[ANNLayerTest]") -{ - // Add function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - input(arma::randu(10, 1)), - target(arma::mat("0")) - { - model = new FFN(); - model->ResetData(input, target); - model->Add(); - model->Add(10, 10); - model->Add(10); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - REQUIRE(CheckGradient(function) <= 1e-4); -}*/ - -/** - * Test that the function that can access the outSize parameter of - * the Add layer works. - * -TEST_CASE("AddLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter : outSize. - Add layer(7); - - // Make sure we can get the parameter successfully. - REQUIRE(layer.OutputSize() == 7); -}*/ - -/** - * Simple constant module test. - * -TEST_CASE("SimpleConstantLayerTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta; - Constant module(10, 3.0); - - // Test the Forward function. - input = arma::zeros(10, 1); - module.Forward(input, output); - REQUIRE(arma::accu(output) == 30.0); - - // Test the Backward function. - module.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 0); - - // Test the forward function. - input = arma::ones(10, 1); - module.Forward(input, output); - REQUIRE(arma::accu(output) == 30.0); - - // Test the backward function. - module.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 0); -}*/ - -/** - * Jacobian constant module test. - * -TEST_CASE("JacobianConstantLayerTest", "[ANNLayerTest]") -{ - for (size_t i = 0; i < 5; ++i) - { - const size_t elements = math::RandInt(2, 1000); - arma::mat input; - input.set_size(elements, 1); - - Constant module(elements, 1.0); - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -}*/ - -/** - * Test that the function that can access the outSize parameter of the - * Constant layer works. - * -TEST_CASE("ConstantLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter : outSize. - Constant layer(7); - - // Make sure we can get the parameter successfully. - REQUIRE(layer.OutSize() == 7); -}*/ - /** * Simple dropout module test. */ @@ -410,90 +202,6 @@ TEST_CASE("NoAlphaDropoutTest", "[ANNLayerTest]") REQUIRE(arma::accu(output) == arma::accu(input)); } -// /** -// * Simple linear module test. -// */ -// TEST_CASE("SimpleLinearLayerTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta; -// Linear<> module(10, 10); -// module.Parameters().randu(); -// module.Reset(); - -// // Test the Forward function. -// input = arma::zeros(10, 1); -// module.Forward(input, output); -// REQUIRE(arma::accu(module.Parameters().submat(100, -// 0, module.Parameters().n_elem - 1, 0)) == -// Approx(arma::accu(output)).epsilon(1e-5)); - -// // Test the Backward function. -// module.Backward(input, input, delta); -// REQUIRE(arma::accu(delta) == 0); -// } - -// /** -// * Jacobian linear module test. -// */ -// TEST_CASE("JacobianLinearLayerTest", "[ANNLayerTest]") -// { -// for (size_t i = 0; i < 5; ++i) -// { -// const size_t inputElements = math::RandInt(2, 1000); -// const size_t outputElements = math::RandInt(2, 1000); - -// arma::mat input; -// input.set_size(inputElements, 1); - -// Linear<> module(inputElements, outputElements); -// module.Parameters().randu(); - -// double error = JacobianTest(module, input); -// REQUIRE(error <= 1e-5); -// } -// } - -// /** -// * Linear layer numerical gradient test. -// */ -// TEST_CASE("GradientLinearLayerTest", "[ANNLayerTest]") -// { -// // Linear function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(10, 1)), -// target(arma::mat("1")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(10, 10); -// model->Add >(10, 2); -// model->Add >(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// arma::mat input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - /** * Simple Linear3D layer test. */ @@ -609,80 +317,6 @@ TEST_CASE("GradientLinear3DLayerTest", "[ANNLayerTest]") REQUIRE(CheckGradient(function) <= 1e-7); } -// /** -// * Simple noisy linear module test. -// */ -// TEST_CASE("SimpleNoisyLinearLayerTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta; -// NoisyLinear<> module(10, 10); -// module.Parameters().randu(); -// module.Reset(); - -// // Test the Backward function. -// module.Backward(input, input, delta); -// REQUIRE(arma::accu(delta) == 0); -// } - -// /** -// * Jacobian noisy linear module test. -// */ -// TEST_CASE("JacobianNoisyLinearLayerTest", "[ANNLayerTest]") -// { -// const size_t inputElements = math::RandInt(2, 1000); -// const size_t outputElements = math::RandInt(2, 1000); - -// arma::mat input; -// input.set_size(inputElements, 1); - -// NoisyLinear<> module(inputElements, outputElements); -// module.Parameters().randu(); - -// double error = JacobianTest(module, input); -// REQUIRE(error <= 1e-5); -// } - -// /** -// * Noisy Linear layer numerical gradient test. -// */ -// TEST_CASE("GradientNoisyLinearLayerTest", "[ANNLayerTest]") -// { -// // Noisy linear function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(10, 1)), -// target(arma::mat("1")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(10, 10); -// model->Add >(10, 2); -// model->Add >(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// arma::mat input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - /** * Simple linear no bias module test. */ @@ -835,1837 +469,6 @@ TEST_CASE("GradientLinearNoBiasLayerTest", "[ANNLayerTest]") REQUIRE(CheckGradient(function) <= 1e-4); } -// /** -// * Jacobian negative log likelihood module test. -// */ -// TEST_CASE("JacobianNegativeLogLikelihoodLayerTest", "[ANNLayerTest]") -// { -// for (size_t i = 0; i < 5; ++i) -// { -// NegativeLogLikelihood module; -// const size_t inputElements = math::RandInt(5, 100); -// arma::mat input; -// RandomInitialization init(0, 1); -// init.Initialize(input, inputElements, 1); - -// arma::mat target(1, 1); -// target(0) = math::RandInt(0, inputElements - 2); - -// double error = JacobianPerformanceTest(module, input, target); -// REQUIRE(error <= 1e-5); -// } -// } - -/** - * Jacobian LeakyReLU module test. - * -TEST_CASE("JacobianLeakyReLULayerTest", "[ANNLayerTest]") -{ - for (size_t i = 0; i < 5; ++i) - { - const size_t inputElements = math::RandInt(2, 1000); - - arma::mat input; - input.set_size(inputElements, 1); - - LeakyReLU module; - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -} -*/ - -/** - * Jacobian FlexibleReLU module test. - * -TEST_CASE("JacobianFlexibleReLULayerTest", "[ANNLayerTest]") -{ - for (size_t i = 0; i < 5; ++i) - { - const size_t inputElements = math::RandInt(2, 1000); - - arma::mat input; - input.set_size(inputElements, 1); - - FlexibleReLU module; - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -} -*/ - -/** - * Flexible ReLU layer numerical gradient test. - * -TEST_CASE("GradientFlexibleReLULayerTest", "[ANNLayerTest]") -{ - // Add function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - input(arma::randu(2, 1)), - target(arma::mat("0")) - { - model = new FFN( - NegativeLogLikelihood(), RandomInitialization(0.1, 0.5)); - - model->ResetData(input, target); - model->Add(2, 2); - model->Add(2, 5); - model->Add(0.05); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - REQUIRE(CheckGradient(function) <= 1e-4); -} -*/ - -/** - * Jacobian MultiplyConstant module test. - * -TEST_CASE("JacobianMultiplyConstantLayerTest", "[ANNLayerTest]") -{ - for (size_t i = 0; i < 5; ++i) - { - const size_t inputElements = math::RandInt(2, 1000); - - arma::mat input; - input.set_size(inputElements, 1); - - MultiplyConstant module(3.0); - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -} -*/ - -/** - * Check whether copying and moving network with MultiplyConstant is working or - * not. - */ -// TEST_CASE("CheckCopyMoveMultiplyConstantTest", "[ANNLayerTest]") -// { -// arma::mat input(2, 1000); -// input.randu(); -// -// arma::mat output1; -// arma::mat output2; -// arma::mat output3; -// arma::mat output4; -// -// MultiplyConstant<> *module1 = new MultiplyConstant<>(3.0); -// module1->Forward(input, output1); -// -// MultiplyConstant<> module2 = *module1; -// delete module1; -// -// module2.Forward(input, output2); -// CheckMatrices(output1, output2); -// -// MultiplyConstant<> *module3 = new MultiplyConstant<>(3.0); -// module3->Forward(input, output3); -// -// MultiplyConstant<> module4(std::move(*module3)); -// delete module3; -// -// module4.Forward(input, output4); -// CheckMatrices(output3, output4); -// } - -/** - * Jacobian HardTanH module test. - * -TEST_CASE("JacobianHardTanHLayerTest", "[ANNLayerTest]") -{ - for (size_t i = 0; i < 5; ++i) - { - const size_t inputElements = math::RandInt(2, 1000); - - arma::mat input; - input.set_size(inputElements, 1); - - HardTanH module; - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -} -*/ - -/** - * Simple select module test. - * -TEST_CASE("SimpleSelectLayerTest", "[ANNLayerTest]") -{ - // TODO: this needs to be adapted - arma::mat outputA, outputB, input, delta; - - input = arma::ones(10, 5); - for (size_t i = 0; i < input.n_cols; ++i) - { - input.col(i) *= i; - } - - // Test the Forward function. - Select moduleA(3); - moduleA.Forward(input, outputA); - REQUIRE(30 == arma::accu(outputA)); - - // Test the Forward function. - Select moduleB(3, 5); - moduleB.Forward(input, outputB); - REQUIRE(15 == arma::accu(outputB)); - - // Test the Backward function. - moduleA.Backward(input, outputA, delta); - REQUIRE(30 == arma::accu(delta)); - - // Test the Backward function. - moduleB.Backward(input, outputA, delta); - REQUIRE(15 == arma::accu(delta)); -} -*/ - -/** - * Test that the functions that can access the parameters of the - * Select layer work. - * -TEST_CASE("SelectLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : index, elements. - Select layer(3, 5); - - // Make sure we can get the parameters successfully. - REQUIRE(layer.Index() == 3); - REQUIRE(layer.NumElements() == 5); -} -*/ - -/** - * Simple join module test. - * -TEST_CASE("SimpleJoinLayerTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta; - input = arma::ones(10, 5); - - // Test the Forward function. - Join module; - module.Forward(input, output); - REQUIRE(50 == arma::accu(output)); - - bool b = output.n_rows == 1 || output.n_cols == 1; - REQUIRE(b == true); - - // Test the Backward function. - module.Backward(input, output, delta); - REQUIRE(50 == arma::accu(delta)); - - b = delta.n_rows == input.n_rows && input.n_cols; - REQUIRE(b == true); -} -*/ - -// /** -// * Simple add merge module test. -// */ -// TEST_CASE("SimpleAddMergeLayerTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta; -// input = arma::ones(10, 1); - -// for (size_t i = 0; i < 5; ++i) -// { -// AddMerge<> module(false, false); -// const size_t numMergeModules = math::RandInt(2, 10); -// for (size_t m = 0; m < numMergeModules; ++m) -// { -// IdentityLayer<> identityLayer; -// identityLayer.Forward(input, identityLayer.OutputParameter()); - -// module.Add >(identityLayer); -// } - -// // Test the Forward function. -// module.Forward(input, output); -// REQUIRE(10 * numMergeModules == arma::accu(output)); - -// // Test the Backward function. -// module.Backward(input, output, delta); -// REQUIRE(arma::accu(output) == arma::accu(delta)); -// } -// } - -// /** -// * Test the LSTM layer with a user defined rho parameter and without. -// */ -// TEST_CASE("LSTMRrhoTest", "[ANNLayerTest]") -// { -// const size_t rho = 5; -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::ones(1, 1, 5); -// RandomInitialization init(0.5, 0.5); - -// // Create model with user defined rho parameter. -// RNN modelA( -// rho, false, NegativeLogLikelihood(), init); -// modelA.Add >(); -// modelA.Add >(1, 10); - -// // Use LSTM layer with rho. -// modelA.Add >(10, 3, rho); -// modelA.Add >(); - -// // Create model without user defined rho parameter. -// RNN modelB( -// rho, false, NegativeLogLikelihood(), init); -// modelB.Add >(); -// modelB.Add >(1, 10); - -// // Use LSTM layer with rho = MAXSIZE. -// modelB.Add >(10, 3); -// modelB.Add >(); - -// ens::StandardSGD opt(0.1, 1, 5, -100, false); -// modelA.Train(input, target, opt); -// modelB.Train(input, target, opt); - -// CheckMatrices(modelB.Parameters(), modelA.Parameters()); -// } - -// /** -// * LSTM layer numerical gradient test. -// */ -// TEST_CASE("GradientLSTMLayerTest", "[ANNLayerTest]") -// { -// // LSTM function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(1, 1, 5)), -// target(arma::ones(1, 1, 5)) -// { -// const size_t rho = 5; - -// model = new RNN(rho); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 10); -// model->Add >(10, 3, rho); -// model->Add >(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// RNN* model; -// arma::cube input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -// /** -// * Test that the functions that can modify and access the parameters of the -// * LSTM layer work. -// */ -// TEST_CASE("LSTMLayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : inSize, outSize, rho. -// LSTM<> layer1(1, 2, 3); -// LSTM<> layer2(1, 2, 4); - -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InSize() == 1); -// REQUIRE(layer1.OutSize() == 2); -// REQUIRE(layer1.Rho() == 3); - -// // Now modify the parameters to match the second layer. -// layer1.Rho() = 4; - -// // Now ensure all the results are the same. -// REQUIRE(layer1.InSize() == layer2.InSize()); -// REQUIRE(layer1.OutSize() == layer2.OutSize()); -// REQUIRE(layer1.Rho() == layer2.Rho()); -// } - -// /** -// * Test the FastLSTM layer with a user defined rho parameter and without. -// */ -// TEST_CASE("FastLSTMRrhoTest", "[ANNLayerTest]") -// { -// const size_t rho = 5; -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::ones(1, 1, 5); -// RandomInitialization init(0.5, 0.5); - -// // Create model with user defined rho parameter. -// RNN modelA( -// rho, false, NegativeLogLikelihood(), init); -// modelA.Add >(); -// modelA.Add >(1, 10); - -// // Use FastLSTM layer with rho. -// modelA.Add >(10, 3, rho); -// modelA.Add >(); - -// // Create model without user defined rho parameter. -// RNN modelB( -// rho, false, NegativeLogLikelihood(), init); -// modelB.Add >(); -// modelB.Add >(1, 10); - -// // Use FastLSTM layer with rho = MAXSIZE. -// modelB.Add >(10, 3); -// modelB.Add >(); - -// ens::StandardSGD opt(0.1, 1, 5, -100, false); -// modelA.Train(input, target, opt); -// modelB.Train(input, target, opt); - -// CheckMatrices(modelB.Parameters(), modelA.Parameters()); -// } - -// /** -// * FastLSTM layer numerical gradient test. -// */ -// TEST_CASE("GradientFastLSTMLayerTest", "[ANNLayerTest]") -// { -// // Fast LSTM function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(1, 1, 5)), -// target(arma::ones(1, 1, 5)) -// { -// const size_t rho = 5; - -// model = new RNN(rho); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 10); -// model->Add >(10, 3, rho); -// model->Add >(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// RNN* model; -// arma::cube input, target; -// } function; - -// // The threshold should be << 0.1 but since the Fast LSTM layer uses an -// // approximation of the sigmoid function the estimated gradient is not -// // correct. -// REQUIRE(CheckGradient(function) <= 0.2); -// } - -// /** -// * Test that the functions that can modify and access the parameters of the -// * Fast LSTM layer work. -// */ -// TEST_CASE("FastLSTMLayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : inSize, outSize, rho. -// FastLSTM<> layer1(1, 2, 3); -// FastLSTM<> layer2(1, 2, 4); - -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InSize() == 1); -// REQUIRE(layer1.OutSize() == 2); -// REQUIRE(layer1.Rho() == 3); - -// // Now modify the parameters to match the second layer. -// layer1.Rho() = 4; - -// // Now ensure all the results are the same. -// REQUIRE(layer1.InSize() == layer2.InSize()); -// REQUIRE(layer1.OutSize() == layer2.OutSize()); -// REQUIRE(layer1.Rho() == layer2.Rho()); -// } - -// /** -// * Check whether copying and moving network with FastLSTM is working or not. -// */ -// TEST_CASE("CheckCopyMoveFastLSTMTest", "[ANNLayerTest]") -// { -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::ones(1, 1, 5); -// const size_t rho = 5; - -// RNN *model1 = -// new RNN(rho); -// model1->ResetData(input, target); -// model1->Add >(); -// model1->Add >(1, 10); -// model1->Add >(10, 3, rho); -// model1->Add >(); - -// RNN *model2 = -// new RNN(rho); -// model2->ResetData(input, target); -// model2->Add >(); -// model2->Add >(1, 10); -// model2->Add >(10, 3, rho); -// model2->Add >(); - -// // Check whether copy constructor is working or not. -// CheckRNNCopyFunction<>(model1, input, target, 1); - -// // Check whether move constructor is working or not. -// CheckRNNMoveFunction<>(model2, input, target, 1); -// } - -// /** -// * Check whether copying and moving network with LSTM is working or not. -// */ -// TEST_CASE("CheckCopyMoveLSTMTest", "[ANNLayerTest]") -// { -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::ones(1, 1, 5); -// const size_t rho = 5; - -// RNN *model1 = -// new RNN(rho); -// model1->ResetData(input, target); -// model1->Add >(); -// model1->Add >(1, 10); -// model1->Add >(10, 3, rho); -// model1->Add >(); - -// RNN *model2 = -// new RNN(rho); -// model2->ResetData(input, target); -// model2->Add >(); -// model2->Add >(1, 10); -// model2->Add >(10, 3, rho); -// model2->Add >(); - -// // Check whether copy constructor is working or not. -// CheckRNNCopyFunction<>(model1, input, target, 1); - -// // Check whether move constructor is working or not. -// CheckRNNMoveFunction<>(model2, input, target, 1); -// } - -// /** -// * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell -// * state. Besides output, the overloaded function provides read access to cell -// * state of the LSTM layer. -// */ -// TEST_CASE("ReadCellStateParamLSTMLayerTest", "[ANNLayerTest]") -// { -// const size_t rho = 5, inputSize = 3, outputSize = 2; - -// // Provide input of all ones. -// arma::cube input = arma::ones(inputSize, outputSize, rho); - -// arma::mat inputGate, forgetGate, outputGate, hidden; -// arma::mat outLstm, cellLstm; - -// // LSTM layer. -// LSTM<> lstm(inputSize, outputSize, rho); -// lstm.Reset(); -// lstm.ResetCell(rho); - -// // Initialize the weights to all ones. -// lstm.Parameters().ones(); - -// arma::mat inputWeight = arma::ones(outputSize, inputSize); -// arma::mat outputWeight = arma::ones(outputSize, outputSize); -// arma::mat bias = arma::ones(outputSize, input.n_cols); -// arma::mat cellCalc = arma::zeros(outputSize, input.n_cols); -// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); - -// for (size_t seqNum = 0; seqNum < rho; ++seqNum) -// { -// // Wrap a matrix around our data to avoid a copy. -// arma::mat stepData(input.slice(seqNum).memptr(), -// input.n_rows, input.n_cols, false, true); - -// // Apply Forward() on LSTM layer. -// lstm.Forward(stepData, // Input. -// outLstm, // Output. -// cellLstm, // Cell state. -// false); // Don't write into the cell state. - -// // Compute the value of cell state and output. -// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); - -// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); - -// // z = tanh(W.dot(x) + W.dot(h) + b). -// hidden = arma::tanh(inputWeight * stepData + -// outputWeight * outCalc + bias); - -// // c = f * c + i * z. -// cellCalc = forgetGate % cellCalc + inputGate % hidden; - -// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); - -// // h = o * tanh(c). -// outCalc = outputGate % arma::tanh(cellCalc); - -// CheckMatrices(outLstm, outCalc, 1e-12); -// CheckMatrices(cellLstm, cellCalc, 1e-12); -// } -// } - -// /** -// * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell -// * state. Besides output, the overloaded function provides write access to cell -// * state of the LSTM layer. -// */ -// TEST_CASE("WriteCellStateParamLSTMLayerTest", "[ANNLayerTest]") -// { -// const size_t rho = 5, inputSize = 3, outputSize = 2; - -// // Provide input of all ones. -// arma::cube input = arma::ones(inputSize, outputSize, rho); - -// arma::mat inputGate, forgetGate, outputGate, hidden; -// arma::mat outLstm, cellLstm; -// arma::mat cellCalc; - -// // LSTM layer. -// LSTM<> lstm(inputSize, outputSize, rho); -// lstm.Reset(); -// lstm.ResetCell(rho); - -// // Initialize the weights to all ones. -// lstm.Parameters().ones(); - -// arma::mat inputWeight = arma::ones(outputSize, inputSize); -// arma::mat outputWeight = arma::ones(outputSize, outputSize); -// arma::mat bias = arma::ones(outputSize, input.n_cols); -// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); - -// for (size_t seqNum = 0; seqNum < rho; ++seqNum) -// { -// // Wrap a matrix around our data to avoid a copy. -// arma::mat stepData(input.slice(seqNum).memptr(), -// input.n_rows, input.n_cols, false, true); - -// if (cellLstm.is_empty()) -// { -// // Set the cell state to zeros. -// cellLstm = arma::zeros(outputSize, input.n_cols); -// cellCalc = arma::zeros(outputSize, input.n_cols); -// } -// else -// { -// // Set the cell state to zeros. -// cellLstm = arma::zeros(cellLstm.n_rows, cellLstm.n_cols); -// cellCalc = arma::zeros(cellCalc.n_rows, cellCalc.n_cols); -// } - -// // Apply Forward() on the LSTM layer. -// lstm.Forward(stepData, // Input. -// outLstm, // Output. -// cellLstm, // Cell state. -// true); // Write into cell state. - -// // Compute the value of cell state and output. -// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); - -// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); - -// // z = tanh(W.dot(x) + W.dot(h) + b). -// hidden = arma::tanh(inputWeight * stepData + -// outputWeight * outCalc + bias); - -// // c = f * c + i * z. -// cellCalc = forgetGate % cellCalc + inputGate % hidden; - -// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); - -// // h = o * tanh(c). -// outCalc = outputGate % arma::tanh(cellCalc); - -// CheckMatrices(outLstm, outCalc, 1e-12); -// CheckMatrices(cellLstm, cellCalc, 1e-12); -// } - -// // Attempting to write empty matrix into cell state. -// lstm.Reset(); -// lstm.ResetCell(rho); -// arma::mat stepData(input.slice(0).memptr(), -// input.n_rows, input.n_cols, false, true); - -// lstm.Forward(stepData, // Input. -// outLstm, // Output. -// cellLstm, // Cell state. -// true); // Write into cell state. - -// for (size_t seqNum = 1; seqNum < rho; ++seqNum) -// { -// arma::mat empty; -// // Should throw error. -// REQUIRE_THROWS_AS(lstm.Forward(stepData, // Input. -// outLstm, // Output. -// empty, // Cell state. -// true), // Write into cell state. -// std::runtime_error); -// } -// } - -// /** -// * Test that the functions that can modify and access the parameters of the -// * GRU layer work. -// */ -// TEST_CASE("GRULayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : inSize, outSize, rho. -// GRU<> layer1(1, 2, 3); -// GRU<> layer2(1, 2, 4); - -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InSize() == 1); -// REQUIRE(layer1.OutSize() == 2); -// REQUIRE(layer1.Rho() == 3); - -// // Now modify the parameters to match the second layer. -// layer1.Rho() = 4; - -// // Now ensure all the results are the same. -// REQUIRE(layer1.InSize() == layer2.InSize()); -// REQUIRE(layer1.OutSize() == layer2.OutSize()); -// REQUIRE(layer1.Rho() == layer2.Rho()); -// } - -// /** -// * Check if the gradients computed by GRU cell are close enough to the -// * approximation of the gradients. -// */ -// TEST_CASE("GradientGRULayerTest", "[ANNLayerTest]") -// { -// // GRU function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(1, 1, 5)), -// target(arma::ones(1, 1, 5)) -// { -// const size_t rho = 5; - -// model = new RNN(rho); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 10); -// model->Add >(10, 3, rho); -// model->Add >(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// arma::mat output; -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// RNN* model; -// arma::cube input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -// /** -// * GRU layer manual forward test. -// */ -// TEST_CASE("ForwardGRULayerTest", "[ANNLayerTest]") -// { -// // This will make it easier to clean memory later. -// GRU<>* gruAlloc = new GRU<>(3, 3, 5); -// GRU<>& gru = *gruAlloc; - -// // Initialize the weights to all ones. -// NetworkInitialization -// networkInit(ConstInitialization(1)); -// networkInit.Initialize(gru.Model(), gru.Parameters()); - -// // Provide input of all ones. -// arma::mat input = arma::ones(3, 1); -// arma::mat output; - -// gru.Forward(input, output); - -// // Compute the z_t gate output. -// arma::mat expectedOutput = arma::ones(3, 1); -// expectedOutput *= -4; -// expectedOutput = arma::exp(expectedOutput); -// expectedOutput = arma::ones(3, 1) / (arma::ones(3, 1) + expectedOutput); -// expectedOutput = (arma::ones(3, 1) - expectedOutput) % expectedOutput; - -// // For the first input the output should be equal to the output of -// // gate z_t as the previous output fed to the cell is all zeros. -// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); - -// expectedOutput = output; - -// gru.Forward(input, output); - -// double s = arma::as_scalar(arma::sum(expectedOutput)); - -// // Compute the value of z_t gate for the second input. -// arma::mat z_t = arma::ones(3, 1); -// z_t *= -(s + 4); -// z_t = arma::exp(z_t); -// z_t = arma::ones(3, 1) / (arma::ones(3, 1) + z_t); - -// // Compute the value of o_t gate for the second input. -// arma::mat o_t = arma::ones(3, 1); -// o_t *= -(arma::as_scalar(arma::sum(expectedOutput % z_t)) + 4); -// o_t = arma::exp(o_t); -// o_t = arma::ones(3, 1) / (arma::ones(3, 1) + o_t); - -// // Expected output for the second input. -// expectedOutput = z_t % expectedOutput + (arma::ones(3, 1) - z_t) % o_t; - -// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); - -// LayerTypes<> layer(gruAlloc); -// boost::apply_visitor(DeleteVisitor(), layer); -// } - -/** - * Simple add merge module test. - */ -// TEST_CASE("SimpleAddMergeLayerTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta; -// input = arma::ones(10, 1); -// -// for (size_t i = 0; i < 5; ++i) -// { -// AddMerge<> module(false, false); -// const size_t numMergeModules = math::RandInt(2, 10); -// for (size_t m = 0; m < numMergeModules; ++m) -// { -// IdentityLayer<> identityLayer; -// identityLayer.Forward(input, identityLayer.OutputParameter()); -// -// module.Add >(identityLayer); -// } -// -// // Test the Forward function. -// module.Forward(input, output); -// REQUIRE(10 * numMergeModules == arma::accu(output)); -// -// // Test the Backward function. -// module.Backward(input, output, delta); -// REQUIRE(arma::accu(output) == arma::accu(delta)); -// } -// } - -/** - * Test the LSTM layer with a user defined rho parameter and without. - */ -// TEST_CASE("LSTMRrhoTest", "[ANNLayerTest]") -// { -// const size_t rho = 5; -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::zeros(1, 1, 5); -// RandomInitialization init(0.5, 0.5); -// -// // Create model with user defined rho parameter. -// RNN modelA( -// rho, false, NegativeLogLikelihood(), init); -// modelA.Add >(); -// modelA.Add >(1, 10); -// -// // Use LSTM layer with rho. -// modelA.Add >(10, 3, rho); -// modelA.Add >(); -// -// // Create model without user defined rho parameter. -// RNN modelB( -// rho, false, NegativeLogLikelihood(), init); -// modelB.Add >(); -// modelB.Add >(1, 10); -// -// // Use LSTM layer with rho = MAXSIZE. -// modelB.Add >(10, 3); -// modelB.Add >(); -// -// ens::StandardSGD opt(0.1, 1, 5, -100, false); -// modelA.Train(input, target, opt); -// modelB.Train(input, target, opt); -// -// CheckMatrices(modelB.Parameters(), modelA.Parameters()); -// } - -/** - * LSTM layer numerical gradient test. - */ -// TEST_CASE("GradientLSTMLayerTest", "[ANNLayerTest]") -// { -// // LSTM function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(1, 1, 5)), -// target(arma::zeros(1, 1, 5)) -// { -// const size_t rho = 5; -// -// model = new RNN(rho); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 10); -// model->Add >(10, 3, rho); -// model->Add >(); -// } -// -// ~GradientFunction() -// { -// delete model; -// } -// -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } -// -// arma::mat& Parameters() { return model->Parameters(); } -// -// RNN* model; -// arma::cube input, target; -// } function; -// -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -/** - * Test that the functions that can modify and access the parameters of the - * LSTM layer work. - */ -// TEST_CASE("LSTMLayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : inSize, outSize, rho. -// LSTM<> layer1(1, 2, 3); -// LSTM<> layer2(1, 2, 4); -// -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InSize() == 1); -// REQUIRE(layer1.OutSize() == 2); -// REQUIRE(layer1.Rho() == 3); -// -// // Now modify the parameters to match the second layer. -// layer1.Rho() = 4; -// -// // Now ensure all the results are the same. -// REQUIRE(layer1.InSize() == layer2.InSize()); -// REQUIRE(layer1.OutSize() == layer2.OutSize()); -// REQUIRE(layer1.Rho() == layer2.Rho()); -// } - -/** - * Test the FastLSTM layer with a user defined rho parameter and without. - */ -// TEST_CASE("FastLSTMRrhoTest", "[ANNLayerTest]") -// { -// const size_t rho = 5; -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::zeros(1, 1, 5); -// RandomInitialization init(0.5, 0.5); -// -// // Create model with user defined rho parameter. -// RNN modelA( -// rho, false, NegativeLogLikelihood(), init); -// modelA.Add >(); -// modelA.Add >(1, 10); -// -// // Use FastLSTM layer with rho. -// modelA.Add >(10, 3, rho); -// modelA.Add >(); -// -// // Create model without user defined rho parameter. -// RNN modelB( -// rho, false, NegativeLogLikelihood(), init); -// modelB.Add >(); -// modelB.Add >(1, 10); -// -// // Use FastLSTM layer with rho = MAXSIZE. -// modelB.Add >(10, 3); -// modelB.Add >(); -// -// ens::StandardSGD opt(0.1, 1, 5, -100, false); -// modelA.Train(input, target, opt); -// modelB.Train(input, target, opt); -// -// CheckMatrices(modelB.Parameters(), modelA.Parameters()); -// } - -/** - * FastLSTM layer numerical gradient test. - */ -// TEST_CASE("GradientFastLSTMLayerTest", "[ANNLayerTest]") -// { -// // Fast LSTM function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(1, 1, 5)), -// target(arma::zeros(1, 1, 5)) -// { -// const size_t rho = 5; -// -// model = new RNN(rho); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 10); -// model->Add >(10, 3, rho); -// model->Add >(); -// } -// -// ~GradientFunction() -// { -// delete model; -// } -// -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } -// -// arma::mat& Parameters() { return model->Parameters(); } -// -// RNN* model; -// arma::cube input, target; -// } function; -// -// // The threshold should be << 0.1 but since the Fast LSTM layer uses an -// // approximation of the sigmoid function the estimated gradient is not -// // correct. -// REQUIRE(CheckGradient(function) <= 0.2); -// } - -/** - * Test that the functions that can modify and access the parameters of the - * Fast LSTM layer work. - */ -// TEST_CASE("FastLSTMLayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : inSize, outSize, rho. -// FastLSTM<> layer1(1, 2, 3); -// FastLSTM<> layer2(1, 2, 4); -// -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InSize() == 1); -// REQUIRE(layer1.OutSize() == 2); -// REQUIRE(layer1.Rho() == 3); -// -// // Now modify the parameters to match the second layer. -// layer1.Rho() = 4; -// -// // Now ensure all the results are the same. -// REQUIRE(layer1.InSize() == layer2.InSize()); -// REQUIRE(layer1.OutSize() == layer2.OutSize()); -// REQUIRE(layer1.Rho() == layer2.Rho()); -// } - -/** - * Check whether copying and moving network with FastLSTM is working or not. - */ -// TEST_CASE("CheckCopyMoveFastLSTMTest", "[ANNLayerTest]") -// { -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::ones(1, 1, 5); -// const size_t rho = 5; -// -// RNN *model1 = -// new RNN(rho); -// model1->ResetData(input, target); -// model1->Add >(); -// model1->Add >(1, 10); -// model1->Add >(10, 3, rho); -// model1->Add >(); -// -// RNN *model2 = -// new RNN(rho); -// model2->ResetData(input, target); -// model2->Add >(); -// model2->Add >(1, 10); -// model2->Add >(10, 3, rho); -// model2->Add >(); -// -// // Check whether copy constructor is working or not. -// CheckRNNCopyFunction<>(model1, input, target, 1); -// -// // Check whether move constructor is working or not. -// CheckRNNMoveFunction<>(model2, input, target, 1); -// } - -/** - * Check whether copying and moving network with LSTM is working or not. - */ -// TEST_CASE("CheckCopyMoveLSTMTest", "[ANNLayerTest]") -// { -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::ones(1, 1, 5); -// const size_t rho = 5; -// -// RNN *model1 = -// new RNN(rho); -// model1->ResetData(input, target); -// model1->Add >(); -// model1->Add >(1, 10); -// model1->Add >(10, 3, rho); -// model1->Add >(); -// -// RNN *model2 = -// new RNN(rho); -// model2->ResetData(input, target); -// model2->Add >(); -// model2->Add >(1, 10); -// model2->Add >(10, 3, rho); -// model2->Add >(); -// -// // Check whether copy constructor is working or not. -// CheckRNNCopyFunction<>(model1, input, target, 1); -// -// // Check whether move constructor is working or not. -// CheckRNNMoveFunction<>(model2, input, target, 1); -// } - -/** - * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell - * state. Besides output, the overloaded function provides read access to cell - * state of the LSTM layer. - */ -// TEST_CASE("ReadCellStateParamLSTMLayerTest", "[ANNLayerTest]") -// { -// const size_t rho = 5, inputSize = 3, outputSize = 2; -// -// // Provide input of all ones. -// arma::cube input = arma::ones(inputSize, outputSize, rho); -// -// arma::mat inputGate, forgetGate, outputGate, hidden; -// arma::mat outLstm, cellLstm; -// -// // LSTM layer. -// LSTM<> lstm(inputSize, outputSize, rho); -// lstm.Reset(); -// lstm.ResetCell(rho); -// -// // Initialize the weights to all ones. -// lstm.Parameters().ones(); -// -// arma::mat inputWeight = arma::ones(outputSize, inputSize); -// arma::mat outputWeight = arma::ones(outputSize, outputSize); -// arma::mat bias = arma::ones(outputSize, input.n_cols); -// arma::mat cellCalc = arma::zeros(outputSize, input.n_cols); -// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); -// -// for (size_t seqNum = 0; seqNum < rho; ++seqNum) -// { -// // Wrap a matrix around our data to avoid a copy. -// arma::mat stepData(input.slice(seqNum).memptr(), -// input.n_rows, input.n_cols, false, true); -// -// // Apply Forward() on LSTM layer. -// lstm.Forward(stepData, // Input. -// outLstm, // Output. -// cellLstm, // Cell state. -// false); // Don't write into the cell state. -// -// // Compute the value of cell state and output. -// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); -// -// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); -// -// // z = tanh(W.dot(x) + W.dot(h) + b). -// hidden = arma::tanh(inputWeight * stepData + -// outputWeight * outCalc + bias); -// -// // c = f * c + i * z. -// cellCalc = forgetGate % cellCalc + inputGate % hidden; -// -// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); -// -// // h = o * tanh(c). -// outCalc = outputGate % arma::tanh(cellCalc); -// -// CheckMatrices(outLstm, outCalc, 1e-12); -// CheckMatrices(cellLstm, cellCalc, 1e-12); -// } -// } - -/** - * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell - * state. Besides output, the overloaded function provides write access to cell - * state of the LSTM layer. - */ -// TEST_CASE("WriteCellStateParamLSTMLayerTest", "[ANNLayerTest]") -// { -// const size_t rho = 5, inputSize = 3, outputSize = 2; -// -// // Provide input of all ones. -// arma::cube input = arma::ones(inputSize, outputSize, rho); -// -// arma::mat inputGate, forgetGate, outputGate, hidden; -// arma::mat outLstm, cellLstm; -// arma::mat cellCalc; -// -// // LSTM layer. -// LSTM<> lstm(inputSize, outputSize, rho); -// lstm.Reset(); -// lstm.ResetCell(rho); -// -// // Initialize the weights to all ones. -// lstm.Parameters().ones(); -// -// arma::mat inputWeight = arma::ones(outputSize, inputSize); -// arma::mat outputWeight = arma::ones(outputSize, outputSize); -// arma::mat bias = arma::ones(outputSize, input.n_cols); -// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); -// -// for (size_t seqNum = 0; seqNum < rho; ++seqNum) -// { -// // Wrap a matrix around our data to avoid a copy. -// arma::mat stepData(input.slice(seqNum).memptr(), -// input.n_rows, input.n_cols, false, true); -// -// if (cellLstm.is_empty()) -// { -// // Set the cell state to zeros. -// cellLstm = arma::zeros(outputSize, input.n_cols); -// cellCalc = arma::zeros(outputSize, input.n_cols); -// } -// else -// { -// // Set the cell state to zeros. -// cellLstm = arma::zeros(cellLstm.n_rows, cellLstm.n_cols); -// cellCalc = arma::zeros(cellCalc.n_rows, cellCalc.n_cols); -// } -// -// // Apply Forward() on the LSTM layer. -// lstm.Forward(stepData, // Input. -// outLstm, // Output. -// cellLstm, // Cell state. -// true); // Write into cell state. -// -// // Compute the value of cell state and output. -// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); -// -// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); -// -// // z = tanh(W.dot(x) + W.dot(h) + b). -// hidden = arma::tanh(inputWeight * stepData + -// outputWeight * outCalc + bias); -// -// // c = f * c + i * z. -// cellCalc = forgetGate % cellCalc + inputGate % hidden; -// -// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); -// -// // h = o * tanh(c). -// outCalc = outputGate % arma::tanh(cellCalc); -// -// CheckMatrices(outLstm, outCalc, 1e-12); -// CheckMatrices(cellLstm, cellCalc, 1e-12); -// } -// -// // Attempting to write empty matrix into cell state. -// lstm.Reset(); -// lstm.ResetCell(rho); -// arma::mat stepData(input.slice(0).memptr(), -// input.n_rows, input.n_cols, false, true); -// -// lstm.Forward(stepData, // Input. -// outLstm, // Output. -// cellLstm, // Cell state. -// true); // Write into cell state. -// -// for (size_t seqNum = 1; seqNum < rho; ++seqNum) -// { -// arma::mat empty; -// // Should throw error. -// REQUIRE_THROWS_AS(lstm.Forward(stepData, // Input. -// outLstm, // Output. -// empty, // Cell state. -// true), // Write into cell state. -// std::runtime_error); -// } -// } - -/** - * Test that the functions that can modify and access the parameters of the - * GRU layer work. - */ -// TEST_CASE("GRULayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : inSize, outSize, rho. -// GRU<> layer1(1, 2, 3); -// GRU<> layer2(1, 2, 4); -// -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InSize() == 1); -// REQUIRE(layer1.OutSize() == 2); -// REQUIRE(layer1.Rho() == 3); -// -// // Now modify the parameters to match the second layer. -// layer1.Rho() = 4; -// -// // Now ensure all the results are the same. -// REQUIRE(layer1.InSize() == layer2.InSize()); -// REQUIRE(layer1.OutSize() == layer2.OutSize()); -// REQUIRE(layer1.Rho() == layer2.Rho()); -// } - -/** - * Check if the gradients computed by GRU cell are close enough to the - * approximation of the gradients. - */ -// TEST_CASE("GradientGRULayerTest", "[ANNLayerTest]") -// { -// // GRU function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(1, 1, 5)), -// target(arma::zeros(1, 1, 5)) -// { -// const size_t rho = 5; -// -// model = new RNN(rho); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 10); -// model->Add >(10, 3, rho); -// model->Add >(); -// } -// -// ~GradientFunction() -// { -// delete model; -// } -// -// double Gradient(arma::mat& gradient) const -// { -// arma::mat output; -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } -// -// arma::mat& Parameters() { return model->Parameters(); } -// -// RNN* model; -// arma::cube input, target; -// } function; -// -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -/** - * GRU layer manual forward test. - */ -// TEST_CASE("ForwardGRULayerTest", "[ANNLayerTest]") -// { -// // This will make it easier to clean memory later. -// GRU<>* gruAlloc = new GRU<>(3, 3, 5); -// GRU<>& gru = *gruAlloc; -// -// // Initialize the weights to all ones. -// NetworkInitialization -// networkInit(ConstInitialization(1)); -// networkInit.Initialize(gru.Model(), gru.Parameters()); -// -// // Provide input of all ones. -// arma::mat input = arma::ones(3, 1); -// arma::mat output; -// -// gru.Forward(input, output); -// -// // Compute the z_t gate output. -// arma::mat expectedOutput = arma::ones(3, 1); -// expectedOutput *= -4; -// expectedOutput = arma::exp(expectedOutput); -// expectedOutput = arma::ones(3, 1) / (arma::ones(3, 1) + expectedOutput); -// expectedOutput = (arma::ones(3, 1) - expectedOutput) % expectedOutput; -// -// // For the first input the output should be equal to the output of -// // gate z_t as the previous output fed to the cell is all zeros. -// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); -// -// expectedOutput = output; -// -// gru.Forward(input, output); -// -// double s = arma::as_scalar(arma::sum(expectedOutput)); -// -// // Compute the value of z_t gate for the second input. -// arma::mat z_t = arma::ones(3, 1); -// z_t *= -(s + 4); -// z_t = arma::exp(z_t); -// z_t = arma::ones(3, 1) / (arma::ones(3, 1) + z_t); -// -// // Compute the value of o_t gate for the second input. -// arma::mat o_t = arma::ones(3, 1); -// o_t *= -(arma::as_scalar(arma::sum(expectedOutput % z_t)) + 4); -// o_t = arma::exp(o_t); -// o_t = arma::ones(3, 1) / (arma::ones(3, 1) + o_t); -// -// // Expected output for the second input. -// expectedOutput = z_t % expectedOutput + (arma::ones(3, 1) - z_t) % o_t; -// -// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); -// -// LayerTypes<> layer(gruAlloc); -// boost::apply_visitor(DeleteVisitor(), layer); -// } - -/** - * Simple concat module test. - * -TEST_CASE("SimpleConcatLayerTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta, error; - - Linear* moduleA = new Linear(10, 10); - moduleA->Parameters().randu(); - moduleA->Reset(); - - Linear* moduleB = new Linear(10, 10); - moduleB->Parameters().randu(); - moduleB->Reset(); - - Concat module; - module.Add(moduleA); - module.Add(moduleB); - - // Test the Forward function. - input = arma::zeros(10, 1); - module.Forward(input, output); - - const double sumModuleA = arma::accu( - moduleA->Parameters().submat( - 100, 0, moduleA->Parameters().n_elem - 1, 0)); - const double sumModuleB = arma::accu( - moduleB->Parameters().submat( - 100, 0, moduleB->Parameters().n_elem - 1, 0)); - REQUIRE(sumModuleA + sumModuleB == - Approx(arma::accu(output.col(0))).epsilon(1e-5)); - - // Test the Backward function. - error = arma::zeros(20, 1); - module.Backward(input, error, delta); - REQUIRE(arma::accu(delta) == 0); -} -*/ - -/** - * Test to check Concat layer along different axes. - * -TEST_CASE("ConcatAlongAxisTest", "[ANNLayerTest]") -{ - arma::mat output, input, error, outputA, outputB; - size_t inputWidth = 4, inputHeight = 4, inputChannel = 2; - size_t outputWidth, outputHeight, outputChannel = 2; - size_t kW = 3, kH = 3; - size_t batch = 1; - - // Using Convolution<> layer as inout to Concat<> layer. - // Compute the output shape of convolution layer. - outputWidth = (inputWidth - kW) + 1; - outputHeight = (inputHeight - kH) + 1; - - input = arma::ones(inputWidth * inputHeight * inputChannel, batch); - - Convolution* moduleA = new Convolution(inputChannel, outputChannel, - kW, kH, 1, 1, 0, 0, inputWidth, inputHeight); - Convolution* moduleB = new Convolution(inputChannel, outputChannel, - kW, kH, 1, 1, 0, 0, inputWidth, inputHeight); - - moduleA->Reset(); - moduleA->Parameters().randu(); - moduleB->Reset(); - moduleB->Parameters().randu(); - - // Compute output of each layer. - moduleA->Forward(input, outputA); - moduleB->Forward(input, outputB); - - arma::cube A(outputA.memptr(), outputWidth, outputHeight, outputChannel); - arma::cube B(outputB.memptr(), outputWidth, outputHeight, outputChannel); - - error = arma::ones(outputWidth * outputHeight * outputChannel * 2, 1); - - for (size_t axis = 0; axis < 3; ++axis) - { - size_t x = 1, y = 1, z = 1; - arma::cube calculatedOut; - if (axis == 0) - { - calculatedOut.set_size(2 * outputWidth, outputHeight, outputChannel); - for (size_t i = 0; i < A.n_slices; ++i) - { - arma::mat aMat = A.slice(i); - arma::mat bMat = B.slice(i); - calculatedOut.slice(i) = arma::join_cols(aMat, bMat); - } - x = 2; - } - if (axis == 1) - { - calculatedOut.set_size(outputWidth, 2 * outputHeight, outputChannel); - for (size_t i = 0; i < A.n_slices; ++i) - { - arma::mat aMat = A.slice(i); - arma::mat bMat = B.slice(i); - calculatedOut.slice(i) = arma::join_rows(aMat, bMat); - } - y = 2; - } - if (axis == 2) - { - calculatedOut = arma::join_slices(A, B); - z = 2; - } - - // Compute output of Concat<> layer. - arma::Row inputSize{outputWidth, outputHeight, outputChannel}; - Concat module(inputSize, axis, true); - module.Add(moduleA); - module.Add(moduleB); - module.Forward(input, output); - arma::cube concatOut(output.memptr(), x * outputWidth, - y * outputHeight, z * outputChannel); - - // Verify if the output reshaped to cubes are similar. - CheckMatrices(concatOut, calculatedOut, 1e-12); - } - delete moduleA; - delete moduleB; -}*/ - -/** - * Test that the function that can access the axis parameter of the - * Concat layer works. - * -TEST_CASE("ConcatLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : inputSize{width, height, channels}, axis, model, run. - arma::Row inputSize{128, 128, 3}; - Concat layer(inputSize, 2, false, true); - - // Make sure we can get the parameters successfully. - REQUIRE(layer.ConcatAxis() == 2); -} -*/ - -/** - * Concat layer numerical gradient test. - */ -// TEST_CASE("GradientConcatLayerTest", "[ANNLayerTest]") -// { -// // Concat function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(10, 1)), -// target(arma::mat("0")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add(); -// model->Add(10, 10); - -// concat = new Concat(true); -// concat->Add(10, 2); -// model->Add(concat); - -// model->Add(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// Concat* concat; -// arma::mat input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -/** - * Simple concatenate module test. - * -TEST_CASE("SimpleConcatenateLayerTest", "[ANNLayerTest]") -{ - arma::mat input = arma::ones(5, 1); - arma::mat output, delta; - - Concatenate module; - module.Concat() = arma::ones(5, 1) * 0.5; - - // Test the Forward function. - module.Forward(input, output); - - REQUIRE(arma::accu(output) == 7.5); - - // Test the Backward function. - module.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 5); -} -*/ - -/** - * Concatenate layer numerical gradient test. - * -TEST_CASE("GradientConcatenateLayerTest", "[ANNLayerTest]") -{ - // Concatenate function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - input(arma::randu(10, 1)), - target(arma::mat("0")) - { - model = new FFN(); - model->ResetData(input, target); - model->Add(); - model->Add(10, 5); - - arma::mat concat = arma::ones(5, 1); - // concatenate = new Concatenate(); - // concatenate->Concat() = concat; - // model->Add(concatenate); - model->Add(concat); - - model->Add(10, 5); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - Concatenate* concatenate; - arma::mat input, target; - } function; - - REQUIRE(CheckGradient(function) <= 1e-4); -} -*/ - -/** - * Simple lookup module test. - * -TEST_CASE("SimpleLookupLayerTest", "[ANNLayerTest]") -{ - const size_t vocabSize = 10; - const size_t embeddingSize = 2; - const size_t seqLength = 3; - const size_t batchSize = 4; - - arma::mat output, input, gy, g, gradient; - - Lookup module(vocabSize, embeddingSize); - module.Parameters().randu(); - - // Test the Forward function. - input = arma::zeros(seqLength, batchSize); - for (size_t i = 0; i < input.n_elem; ++i) - { - int token = math::RandInt(1, vocabSize); - input(i) = token; - } - - module.Forward(input, output); - for (size_t i = 0; i < batchSize; ++i) - { - // The Lookup module uses index - 1 for the cols. - const double outputSum = arma::accu(module.Parameters().cols( - arma::conv_to::from(input.col(i)) - 1)); - - REQUIRE(std::fabs(outputSum - arma::accu(output.col(i))) <= 1e-5); - } - - // Test the Gradient function. - arma::mat error = 0.01 * arma::randu(embeddingSize * seqLength, batchSize); - module.Gradient(input, error, gradient); - - REQUIRE(std::fabs(arma::accu(error) - arma::accu(gradient)) <= 1e-07); -} -*/ - -/** - * Lookup layer numerical gradient test. - * -TEST_CASE("GradientLookupLayerTest", "[ANNLayerTest]") -{ - // Lookup function gradient instantiation. - struct GradientFunction - { - GradientFunction() - { - input.set_size(seqLength, batchSize); - for (size_t i = 0; i < input.n_elem; ++i) - { - input(i) = math::RandInt(1, vocabSize); - } - target = arma::zeros(vocabSize, batchSize); - for (size_t i = 0; i < batchSize; ++i) - { - const size_t targetWord = math::RandInt(1, vocabSize); - target(targetWord, i) = 1; - } - - model = new FFN, GlorotInitialization>(BCELoss<>(1e-10, false)); - model->ResetData(input, target); - model->Add(vocabSize, embeddingSize); - model->Add(embeddingSize * seqLength, vocabSize); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, batchSize); - model->Gradient(model->Parameters(), 0, gradient, batchSize); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN, GlorotInitialization>* model; - arma::mat input, target; - - const size_t seqLength = 10; - const size_t embeddingSize = 8; - const size_t vocabSize = 20; - const size_t batchSize = 4; - } function; - - REQUIRE(CheckGradient(function) <= 1e-6); -} -*/ - -/** - * Test that the functions that can access the parameters of the - * Lookup layer work. - * -TEST_CASE("LookupLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : vocabSize, embedingSize. - Lookup layer(100, 8); - - // Make sure we can get the parameters successfully. - REQUIRE(layer.VocabSize() == 100); - REQUIRE(layer.EmbeddingSize() == 8); -} -*/ - /** * Simple LogSoftMax module test. */ @@ -2689,1886 +492,6 @@ TEST_CASE("SimpleLogSoftmaxLayerTest", "[ANNLayerTest]") Approx(0.0).margin(1e-3)); } -/** - * Simple Softmax module test. - * -TEST_CASE("SimpleSoftmaxLayerTest", "[ANNLayerTest]") -{ - arma::mat input, output, gy, g; - Softmax module; - - // Test the forward function. - input = arma::mat("1.7; 3.6"); - module.Forward(input, output); - REQUIRE(arma::accu(arma::abs(arma::mat("0.130108; 0.869892") - output)) == - Approx(0.0).margin(1e-4)); - - // Test the backward function. - gy = arma::zeros(input.n_rows, input.n_cols); - gy(0) = 1; - module.Backward(output, gy, g); - REQUIRE(arma::accu(arma::abs(arma::mat("0.11318; -0.11318") - g)) == - Approx(0.0).margin(1e-04)); -} -*/ - -/** - * Softmax layer numerical gradient test. - * -TEST_CASE("GradientSoftmaxTest", "[ANNLayerTest]") -{ - // Softmax function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - input(arma::randu(10, 1)), - target(arma::mat("1; 0")) - { - model = new FFN; - model->ResetData(input, target); - model->Add(10, 10); - model->Add(); - model->Add(10, 2); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - REQUIRE(CheckGradient(function) <= 1e-4); -} -*/ - -/** - * Simple test for the NearestInterpolation layer - * -TEST_CASE("SimpleNearestInterpolationLayerTest", "[ANNLayerTest]") -{ - // Tested output against torch.nn.Upsample(mode="nearest"). - arma::mat input, output, unzoomedOutput, expectedOutput; - size_t inRowSize = 2; - size_t inColSize = 2; - size_t outRowSize = 5; - size_t outColSize = 7; - size_t depth = 1; - input.zeros(inRowSize * inColSize * depth, 1); - input[0] = 1.0; - input[1] = 3.0; - input[2] = 2.0; - input[3] = 4.0; - NearestInterpolation<> layer(inRowSize, inColSize, outRowSize, - outColSize, depth); - - expectedOutput << 1.0000 << 1.0000 << 1.0000 << 1.0000 << 2.0000 - << 2.0000 << 2.0000 << arma::endr - << 1.0000 << 1.0000 << 1.0000 << 1.0000 << 2.0000 - << 2.0000 << 2.0000 << arma::endr - << 1.0000 << 1.0000 << 1.0000 << 1.0000 << 2.0000 - << 2.0000 << 2.0000 << arma::endr - << 3.0000 << 3.0000 << 3.0000 << 3.0000 << 4.0000 - << 4.0000 << 4.0000 << arma::endr - << 3.0000 << 3.0000 << 3.0000 << 3.0000 << 4.0000 - << 4.0000 << 4.0000 << arma::endr; - expectedOutput.reshape(35, 1); - - layer.Forward(input, output); - CheckMatrices(output - expectedOutput, - arma::zeros(output.n_rows), 1e-4); - - expectedOutput.clear(); - expectedOutput << 12.0000 << 18.0000 << arma::endr - << 24.0000 << 24.0000 << arma::endr; - expectedOutput.reshape(4, 1); - layer.Backward(output, output, unzoomedOutput); - CheckMatrices(unzoomedOutput - expectedOutput, - arma::zeros(input.n_rows), 1e-4); - - arma::mat input1, output1, unzoomedOutput1, expectedOutput1; - inRowSize = 2; - inColSize = 3; - outRowSize = 17; - outColSize = 23; - input1 << 1 << 2 << 3 << arma::endr - << 4 << 5 << 6 << arma::endr; - input1.reshape(6, 1); - NearestInterpolation<> layer1(inRowSize, inColSize, outRowSize, - outColSize, depth); - - layer1.Forward(input1, output1); - layer1.Backward(output1, output1, unzoomedOutput1); - - REQUIRE(arma::accu(output1) - 1317.00 == Approx(0.0).margin(1e-05)); - REQUIRE(arma::accu(unzoomedOutput1) - 1317.00 == - Approx(0.0).margin(1e-05)); -} -*/ - -/* - * Simple test for the BilinearInterpolation layer - * -TEST_CASE("SimpleBilinearInterpolationLayerTest", "[ANNLayerTest]") -{ - // Tested output against tensorflow.image.resize_bilinear() - arma::mat input, output, unzoomedOutput, expectedOutput; - size_t inRowSize = 2; - size_t inColSize = 2; - size_t outRowSize = 5; - size_t outColSize = 5; - size_t depth = 1; - input.zeros(inRowSize * inColSize * depth, 1); - input[0] = 1.0; - input[1] = input[2] = 2.0; - input[3] = 3.0; - BilinearInterpolation layer(inRowSize, inColSize, outRowSize, outColSize, - depth); - expectedOutput = arma::mat("1.0000 1.4000 1.8000 2.0000 2.0000 \ - 1.4000 1.8000 2.2000 2.4000 2.4000 \ - 1.8000 2.2000 2.6000 2.8000 2.8000 \ - 2.0000 2.4000 2.8000 3.0000 3.0000 \ - 2.0000 2.4000 2.8000 3.0000 3.0000"); - expectedOutput.reshape(25, 1); - layer.Forward(input, output); - CheckMatrices(output - expectedOutput, arma::zeros(output.n_rows), 1e-12); - - expectedOutput = arma::mat("1.0000 1.9000 1.9000 2.8000"); - expectedOutput.reshape(4, 1); - layer.Backward(output, output, unzoomedOutput); - CheckMatrices(unzoomedOutput - expectedOutput, - arma::zeros(input.n_rows), 1e-12); -} -*/ - -/** - * Test that the functions that can modify and access the parameters of the - * Bilinear Interpolation layer work. - * -TEST_CASE("BilinearInterpolationLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : inRowSize, inColSize, outRowSize, outColSize, depth. - BilinearInterpolation layer1(1, 2, 3, 4, 5); - BilinearInterpolation layer2(2, 3, 4, 5, 6); - - // Make sure we can get the parameters successfully. - REQUIRE(layer1.InRowSize() == 1); - REQUIRE(layer1.InColSize() == 2); - REQUIRE(layer1.OutRowSize() == 3); - REQUIRE(layer1.OutColSize() == 4); - REQUIRE(layer1.InDepth() == 5); - - // Now modify the parameters to match the second layer. - layer1.InRowSize() = 2; - layer1.InColSize() = 3; - layer1.OutRowSize() = 4; - layer1.OutColSize() = 5; - layer1.InDepth() = 6; - - // Now ensure all results are the same. - REQUIRE(layer1.InRowSize() == layer2.InRowSize()); - REQUIRE(layer1.InColSize() == layer2.InColSize()); - REQUIRE(layer1.OutRowSize() == layer2.OutRowSize()); - REQUIRE(layer1.OutColSize() == layer2.OutColSize()); - REQUIRE(layer1.InDepth() == layer2.InDepth()); -} -*/ - -/* - * Simple test for the BicubicInterpolation layer. - * -TEST_CASE("SimpleBicubicInterpolationLayerTest", "[ANNLayerTest]") -{ - // Tested output against torch.nn.Upsample(mode="bicubic"). - // Test case with square input with rectangular output. - arma::mat input, output, unzoomedOutput, expectedOutput; - size_t inRowSize = 2; - size_t inColSize = 2; - size_t outRowSize = 5; - size_t outColSize = 7; - size_t depth = 1; - input.zeros(inRowSize * inColSize * depth, 1); - - input << 10 << 20 << arma::endr - << 30 << 40 << arma::endr; - input.reshape(4, 1); - BicubicInterpolation<> layer(inRowSize, inColSize, outRowSize, - outColSize, depth); - - expectedOutput << 6.68803935860 << 7.33308309038 << 9.69733236152 - << 12.79500000000 << 15.89266763848 << 18.25691690962 - << 18.90196064140 << arma::endr - << 10.53303935860 << 11.17808309038 << 13.54233236152 - << 16.64000000000 << 19.73766763848 << 22.10191690962 - << 22.74696064140 << arma::endr - << 18.89303935860 << 19.53808309038 << 21.90233236152 - << 25.00000000000 << 28.09766763848 << 30.46191690962 - << 31.10696064140 << arma::endr - << 27.25303935860 << 27.89808309038 << 30.26233236152 - << 33.36000000000 << 36.45766763848 << 38.82191690962 - << 39.46696064140 << arma::endr - << 31.09803935860 << 31.74308309038 << 34.10733236152 - << 37.20500000000 << 40.30266763848 << 42.66691690962 - << 43.31196064140 << arma::endr; - expectedOutput.reshape(35, 1); - layer.Forward(input, output); - - CheckMatrices(output, expectedOutput, 1e-6); - - expectedOutput.clear(); - expectedOutput << 103.79040654914 << 180.51345595086 << arma::endr - << 256.98654404914 << 333.70959345086 << arma::endr; - expectedOutput.reshape(4, 1); - - layer.Backward(output, output, unzoomedOutput); - - CheckMatrices(unzoomedOutput, expectedOutput, 1e-6); - - // Tested output against torch.nn.Upsample(mode="bicubic"). - // Test case with rectangular input with rectangular output. - arma::mat input1, output1, unzoomedOutput1, expectedOutput1, expectedUnzoomed; - - inRowSize = 2; - inColSize = 3; - outRowSize = 5; - outColSize = 7; - depth = 1; - input1.zeros(inRowSize * inColSize * depth, 1); - - input1 << 10 << 20 << 30 << arma::endr - << 40 << 50 << 60 << arma::endr; - input1.reshape(6, 1); - - BicubicInterpolation<> layer1(inRowSize, inColSize, outRowSize, - outColSize, depth); - - expectedOutput1 << 5.59920553936 << 7.77121720117 << 11.44468658892 - << 16.69250000000 << 21.94031341108 << 25.61378279883 - << 27.78579446064 << arma::endr - << 11.36670553936 << 13.53871720117 << 17.21218658892 - << 22.46000000000 << 27.70781341108 << 31.38128279883 - << 33.55329446064 << arma::endr - << 23.90670553936 << 26.07871720117 << 29.75218658892 - << 35.00000000000 << 40.24781341108 << 43.92128279883 - << 46.09329446064 << arma::endr - << 36.44670553936 << 38.61871720117 << 42.29218658892 - << 47.54000000000 << 52.78781341108 << 56.46128279883 - << 58.63329446064 << arma::endr - << 42.21420553936 << 44.38621720117 << 48.05968658892 - << 53.30750000000 << 58.55531341108 << 62.22878279883 - << 64.40079446064 << arma::endr; - expectedOutput1.reshape(35, 1); - layer1.Forward(input1, output1); - - CheckMatrices(output1, expectedOutput1, 1e-6); - - expectedUnzoomed << 67.65674505130 << 132.29729646501 - << 182.75175223368 << arma::endr - << 218.01355388877 << 291.17209129009 - << 333.10856107115 << arma::endr; - expectedUnzoomed.reshape(6, 1); - - layer1.Backward(output1, output1, unzoomedOutput1); - CheckMatrices(unzoomedOutput1, expectedUnzoomed, 1e-6); -} -*/ - -// /** -// * Tests the BatchNorm Layer, compares the layers parameters with -// * the values from another implementation. -// * Link to the implementation - http://cthorey.github.io./backpropagation/ -// */ -// TEST_CASE("BatchNormTest", "[ANNLayerTest]") -// { -// arma::mat input, output; -// input << 5.1 << 3.5 << 1.4 << arma::endr -// << 4.9 << 3.0 << 1.4 << arma::endr -// << 4.7 << 3.2 << 1.3 << arma::endr; - -// // BatchNorm layer with average parameter set to true. -// BatchNorm<> model(input.n_rows); -// model.Reset(); - -// // BatchNorm layer with average parameter set to false. -// BatchNorm<> model2(input.n_rows, 1e-5, false); -// model2.Reset(); - -// // Non-Deteministic Forward Pass Test. -// model.Deterministic() = false; -// model.Forward(input, output); - -// // Value calculates using torch.nn.BatchNorm2d(momentum = None). -// arma::mat result; -// result = { { 1.1658, 0.1100, -1.2758 }, -// { 1.2579, -0.0699, -1.1880}, -// { 1.1737, 0.0958, -1.2695 } }; - -// CheckMatrices(output, result, 1e-1); - -// model2.Forward(input, output); -// CheckMatrices(output, result, 1e-1); -// result.clear(); - -// // Values calculated using torch.nn.BatchNorm2d(momentum = None). -// output = model.TrainingMean(); -// result = arma::mat({ 3.33333333, 3.1, 3.06666666 }).t(); - -// CheckMatrices(output, result, 1e-1); - -// // Values calculated using torch.nn.BatchNorm2d(). -// output = model2.TrainingMean(); -// result = arma::mat({ 0.3333, 0.3100, 0.3067 }).t(); - -// CheckMatrices(output, result, 1e-1); -// result.clear(); - - // Values calculated using torch.nn.BatchNorm2d(momentum = None). -// output = model.TrainingVariance(); -// result = arma::mat({ 3.4433, 3.0700, 2.9033 }).t(); - -// CheckMatrices(output, result, 1e-1); -// result.clear(); - - // Values calculated using torch.nn.BatchNorm2d(). -// output = model2.TrainingVariance(); -// result = arma::mat({ 1.2443, 1.2070, 1.1903 }).t(); - -// CheckMatrices(output, result, 1e-1); -// result.clear(); - -// // Deterministic Forward Pass test. -// model.Deterministic() = true; -// model.Forward(input, output); - - // Values calculated using torch.nn.BatchNorm2d(momentum = None). -// result = { { 0.9521, 0.0898, -1.0419 }, -// { 1.0273, -0.0571, -0.9702 }, -// { 0.9586, 0.0783, -1.0368 } }; - -// CheckMatrices(output, result, 1e-1); - -// // Values calculated using torch.nn.BatchNorm2d(). -// model2.Deterministic() = true; -// model2.Forward(input, output); - -// result = { { 4.2731, 2.8388, 0.9562 }, -// { 4.1779, 2.4485, 0.9921 }, -// { 4.0268, 2.6519, 0.9105 } }; -// -// CheckMatrices(output, result, 1e-1); -// } - -// /** -// * BatchNorm layer numerical gradient test. -// */ -// TEST_CASE("GradientBatchNormTest", "[ANNLayerTest]") -// { -// bool pass = false; -// for (size_t trial = 0; trial < 10; trial++) -// { -// // Add function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randn(32, 2048)), -// target(arma::zeros(1, 2048)) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(32, 4); -// model->Add >(4); -// model->Add>(4, 2); -// model->Add >(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 2048, false); -// model->Gradient(model->Parameters(), 0, gradient, 2048); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// arma::mat input, target; -// } function; - -// double gradient = CheckGradient(function); -// if (gradient < 2e-1) -// { -// pass = true; -// break; -// } -// } - -// REQUIRE(pass); -// } - -// /** -// * Test that the functions that can access the parameters of the -// * Batch Norm layer work. -// */ -// TEST_CASE("BatchNormLayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : size, eps. -// BatchNorm<> layer(7, 1e-3); - -// // Make sure we can get the parameters successfully. -// REQUIRE(layer.InputSize() == 7); -// REQUIRE(layer.Epsilon() == 1e-3); - -// arma::mat runningMean(7, 1, arma::fill::randn); -// arma::mat runningVariance(7, 1, arma::fill::randn); - -// layer.TrainingVariance() = runningVariance; -// layer.TrainingMean() = runningMean; -// CheckMatrices(layer.TrainingVariance(), runningVariance); -// CheckMatrices(layer.TrainingMean(), runningMean); -// } - -/** - * VirtualBatchNorm layer numerical gradient test. - * -TEST_CASE("GradientVirtualBatchNormTest", "[ANNLayerTest]") -{ - // Add function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - input(arma::randn(5, 256)), - target(arma::zeros(1, 256)) - { - arma::mat referenceBatch = arma::mat(input.memptr(), input.n_rows, 4); - - model = new FFN(); - model->ResetData(input, target); - model->Add(); - model->Add(5, 5); - model->Add(referenceBatch, 5); - model->Add(5, 2); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 16, false); - model->Gradient(model->Parameters(), 0, gradient, 16); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - REQUIRE(CheckGradient(function) <= 1e-4); -} -*/ - -/** - * Test that the functions that can modify and access the parameters of the - * Virtual Batch Norm layer work. - * -TEST_CASE("VirtualBatchNormLayerParametersTest", "[ANNLayerTest]") -{ - arma::mat input = arma::randn(5, 16); - arma::mat referenceBatch = arma::mat(input.memptr(), input.n_rows, 4); - - // Parameter order : referenceBatch, size, eps. - VirtualBatchNorm layer(referenceBatch, 5, 1e-3); - - // Make sure we can get the parameters successfully. - REQUIRE(layer.InSize() == 5); - REQUIRE(layer.Epsilon() == 1e-3); -} -*/ - -// /** -// * MiniBatchDiscrimination layer numerical gradient test. -// */ -// TEST_CASE("MiniBatchDiscriminationTest", "[ANNLayerTest]") -// { -// // Add function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randn(5, 4)), -// target(arma::zeros(1, 4)) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(5, 5); -// model->Add >(5, 10, 16); -// model->Add >(10, 2); -// model->Add >(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// return model->EvaluateWithGradient(model->Parameters(), 0, gradient, 4); -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// arma::mat input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -/** - * Simple Transposed Convolution layer test. - * -TEST_CASE("SimpleTransposedConvolutionLayerTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta; - - TransposedConvolution module1(1, 1, 3, 3, 1, 1, 0, 0, 4, 4, 6, 6); - // Test the forward function. - input = arma::linspace(0, 15, 16); - module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); - module1.Parameters()(0) = 1.0; - module1.Parameters()(8) = 2.0; - module1.Reset(); - module1.Forward(input, output); - // Value calculated using tensorflow.nn.conv2d_transpose() - REQUIRE(arma::accu(output) == 360.0); - - // Test the backward function. - module1.Backward(input, output, delta); - // Value calculated using tensorflow.nn.conv2d() - REQUIRE(arma::accu(delta) == 720.0); - - TransposedConvolution module2(1, 1, 4, 4, 1, 1, 1, 1, 5, 5, 6, 6); - // Test the forward function. - input = arma::linspace(0, 24, 25); - module2.Parameters() = arma::mat(16 + 1, 1, arma::fill::zeros); - module2.Parameters()(0) = 1.0; - module2.Parameters()(3) = 1.0; - module2.Parameters()(6) = 1.0; - module2.Parameters()(9) = 1.0; - module2.Parameters()(12) = 1.0; - module2.Parameters()(15) = 2.0; - module2.Reset(); - module2.Forward(input, output); - // Value calculated using torch.nn.functional.conv_transpose2d() - REQUIRE(arma::accu(output) == 1512.0); - - // Test the backward function. - module2.Backward(input, output, delta); - // Value calculated using torch.nn.functional.conv2d() - REQUIRE(arma::accu(delta) == 6504.0); - - TransposedConvolution module3(1, 1, 3, 3, 1, 1, 1, 1, 5, 5, 5, 5); - // Test the forward function. - input = arma::linspace(0, 24, 25); - module3.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); - module3.Parameters()(1) = 2.0; - module3.Parameters()(2) = 4.0; - module3.Parameters()(3) = 3.0; - module3.Parameters()(8) = 1.0; - module3.Reset(); - module3.Forward(input, output); - // Value calculated using torch.nn.functional.conv_transpose2d() - REQUIRE(arma::accu(output) == 2370.0); - - // Test the backward function. - module3.Backward(input, output, delta); - // Value calculated using torch.nn.functional.conv2d() - REQUIRE(arma::accu(delta) == 19154.0); - - TransposedConvolution module4(1, 1, 3, 3, 1, 1, 0, 0, 5, 5, 7, 7); - // Test the forward function. - input = arma::linspace(0, 24, 25); - module4.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); - module4.Parameters()(2) = 2.0; - module4.Parameters()(4) = 4.0; - module4.Parameters()(6) = 6.0; - module4.Parameters()(8) = 8.0; - module4.Reset(); - module4.Forward(input, output); - // Value calculated using torch.nn.functional.conv_transpose2d() - REQUIRE(arma::accu(output) == 6000.0); - - // Test the backward function. - module4.Backward(input, output, delta); - // Value calculated using torch.nn.functional.conv2d() - REQUIRE(arma::accu(delta) == 86208.0); - - TransposedConvolution module5(1, 1, 3, 3, 2, 2, 0, 0, 2, 2, 5, 5); - // Test the forward function. - input = arma::linspace(0, 3, 4); - module5.Parameters() = arma::mat(25 + 1, 1, arma::fill::zeros); - module5.Parameters()(2) = 8.0; - module5.Parameters()(4) = 6.0; - module5.Parameters()(6) = 4.0; - module5.Parameters()(8) = 2.0; - module5.Reset(); - module5.Forward(input, output); - // Value calculated using torch.nn.functional.conv_transpose2d() - REQUIRE(arma::accu(output) == 120.0); - - // Test the backward function. - module5.Backward(input, output, delta); - // Value calculated using torch.nn.functional.conv2d() - REQUIRE(arma::accu(delta) == 960.0); - - TransposedConvolution module6(1, 1, 3, 3, 2, 2, 1, 1, 3, 3, 5, 5); - // Test the forward function. - input = arma::linspace(0, 8, 9); - module6.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); - module6.Parameters()(0) = 8.0; - module6.Parameters()(3) = 6.0; - module6.Parameters()(6) = 2.0; - module6.Parameters()(8) = 4.0; - module6.Reset(); - module6.Forward(input, output); - // Value calculated using torch.nn.functional.conv_transpose2d() - REQUIRE(arma::accu(output) == 410.0); - - // Test the backward function. - module6.Backward(input, output, delta); - // Value calculated using torch.nn.functional.conv2d() - REQUIRE(arma::accu(delta) == 4444.0); - - TransposedConvolution module7(1, 1, 3, 3, 2, 2, 1, 1, 3, 3, 6, 6); - // Test the forward function. - input = arma::linspace(0, 8, 9); - module7.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); - module7.Parameters()(0) = 8.0; - module7.Parameters()(2) = 6.0; - module7.Parameters()(4) = 2.0; - module7.Parameters()(8) = 4.0; - module7.Reset(); - module7.Forward(input, output); - // Value calculated using torch.nn.functional.conv_transpose2d() - REQUIRE(arma::accu(output) == 606.0); - - module7.Backward(input, output, delta); - // Value calculated using torch.nn.functional.conv2d() - REQUIRE(arma::accu(delta) == 7732.0); -} -*/ - -/** - * Transposed Convolution layer numerical gradient test. - * -TEST_CASE("GradientTransposedConvolutionLayerTest", "[ANNLayerTest]") -{ - // Add function gradient instantiation. - // To make this test robust, check it five times. - bool pass = false; - for (size_t trial = 0; trial < 5; trial++) - { - struct GradientFunction - { - GradientFunction() : - input(arma::linspace(0, 35, 36)), - target(arma::mat("0")) - { - model = new FFN(); - model->ResetData(input, target); - model->Add(1, 1, 3, 3, 2, 2, 1, 1, 6, 6, 12, 12); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - if (CheckGradient(function) < 1e-3) - { - pass = true; - break; - } - } - REQUIRE(pass == true); -} -*/ - -/** - * Simple MultiplyMerge module test. - * -TEST_CASE("SimpleMultiplyMergeLayerTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta; - input = arma::ones(10, 1); - - for (size_t i = 0; i < 5; ++i) - { - MultiplyMerge module(false, false); - const size_t numMergeModules = math::RandInt(2, 10); - for (size_t m = 0; m < numMergeModules; ++m) - { - IdentityLayer* identityLayer = new IdentityLayer(); - identityLayer->Forward(input, identityLayer->OutputParameter()); - - module.Add(identityLayer); - } - - // Test the Forward function. - module.Forward(input, output); - REQUIRE(10 == arma::accu(output)); - - // Test the Backward function. - module.Backward(input, output, delta); - REQUIRE(arma::accu(output) == arma::accu(delta)); - } -} -*/ - -/** - * Check whether copying and moving network with MultiplyMerge is working or - * not. - */ -// TEST_CASE("CheckCopyMoveMultiplyMergeTest", "[ANNLayerTest]") -// { -// arma::mat input(10, 1); -// input.randu(); -// -// arma::mat output1; -// arma::mat output2; -// arma::mat output3; -// arma::mat output4; -// -// const size_t numMergeModules = math::RandInt(2, 10); -// -// MultiplyMerge<> *module1 = new MultiplyMerge<>(true, false); -// for (size_t m = 0; m < numMergeModules; ++m) -// { -// IdentityLayer<> identityLayer; -// identityLayer.Forward(input, identityLayer.OutputParameter()); -// -// module1->Add >(identityLayer); -// } -// -// module1->Forward(input, output1); -// -// MultiplyMerge<> module2 = *module1; -// delete module1; -// -// module2.Forward(input, output2); -// CheckMatrices(output1, output2); -// -// MultiplyMerge<> *module3 = new MultiplyMerge<>(true, false); -// for (size_t m = 0; m < numMergeModules; ++m) -// { -// IdentityLayer<> identityLayer; -// identityLayer.Forward(input, identityLayer.OutputParameter()); -// -// module3->Add >(identityLayer); -// } -// module3->Forward(input, output3); -// -// MultiplyMerge<> module4(std::move(*module3)); -// delete module3; -// -// module4.Forward(input, output4); -// CheckMatrices(output3, output4); -// } - -// /** -// * Simple Atrous Convolution layer test. -// */ -// TEST_CASE("SimpleAtrousConvolutionLayerTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta; - -// AtrousConvolution<> module1(1, 1, 3, 3, 1, 1, 0, 0, 7, 7, 2, 2); -// // Test the Forward function. -// input = arma::linspace(0, 48, 49); -// module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); -// module1.Parameters()(0) = 1.0; -// module1.Parameters()(8) = 2.0; -// module1.Reset(); -// module1.Forward(input, output); -// // Value calculated using tensorflow.nn.atrous_conv2d() -// REQUIRE(arma::accu(output) == 792.0); - -// // Test the Backward function. -// module1.Backward(input, output, delta); -// REQUIRE(arma::accu(delta) == 2376); - -// AtrousConvolution<> module2(1, 1, 3, 3, 2, 2, 0, 0, 7, 7, 2, 2); -// // Test the forward function. -// input = arma::linspace(0, 48, 49); -// module2.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); -// module2.Parameters()(0) = 1.0; -// module2.Parameters()(3) = 1.0; -// module2.Parameters()(6) = 1.0; -// module2.Reset(); -// module2.Forward(input, output); -// // Value calculated using tensorflow.nn.conv2d() -// REQUIRE(arma::accu(output) == 264.0); - -// // Test the backward function. -// module2.Backward(input, output, delta); -// REQUIRE(arma::accu(delta) == 792.0); -// } - -// /** -// * Atrous Convolution layer numerical gradient test. -// */ -// TEST_CASE("GradientAtrousConvolutionLayerTest", "[ANNLayerTest]") -// { -// // Add function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::linspace(0, 35, 36)), -// target(arma::mat("0")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 1, 3, 3, 1, 1, 0, 0, 6, 6, 2, 2); -// model->Add >(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// arma::mat input, target; -// } function; - -// // TODO: this tolerance seems far higher than necessary. The implementation -// // should be checked. -// REQUIRE(CheckGradient(function) <= 0.2); -// } - -// /** -// * Test the functions to access and modify the parameters of the -// * AtrousConvolution layer. -// */ -// TEST_CASE("AtrousConvolutionLayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order for the constructor: inSize, outSize, kW, kH, dW, dH, padW, -// // padH, inputWidth, inputHeight, dilationW, dilationH, paddingType ("none"). -// AtrousConvolution<> layer1(1, 2, 3, 4, 5, 6, std::make_tuple(7, 8), -// std::make_tuple(9, 10), 11, 12, 13, 14); -// AtrousConvolution<> layer2(2, 3, 4, 5, 6, 7, std::make_tuple(8, 9), -// std::make_tuple(10, 11), 12, 13, 14, 15); - -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InputWidth() == 11); -// REQUIRE(layer1.InputHeight() == 12); -// REQUIRE(layer1.KernelWidth() == 3); -// REQUIRE(layer1.KernelHeight() == 4); -// REQUIRE(layer1.StrideWidth() == 5); -// REQUIRE(layer1.StrideHeight() == 6); -// REQUIRE(layer1.Padding().PadHTop() == 9); -// REQUIRE(layer1.Padding().PadHBottom() == 10); -// REQUIRE(layer1.Padding().PadWLeft() == 7); -// REQUIRE(layer1.Padding().PadWRight() == 8); -// REQUIRE(layer1.DilationWidth() == 13); -// REQUIRE(layer1.DilationHeight() == 14); - -// // Now modify the parameters to match the second layer. -// layer1.InputWidth() = 12; -// layer1.InputHeight() = 13; -// layer1.KernelWidth() = 4; -// layer1.KernelHeight() = 5; -// layer1.StrideWidth() = 6; -// layer1.StrideHeight() = 7; -// layer1.Padding().PadHTop() = 10; -// layer1.Padding().PadHBottom() = 11; -// layer1.Padding().PadWLeft() = 8; -// layer1.Padding().PadWRight() = 9; -// layer1.DilationWidth() = 14; -// layer1.DilationHeight() = 15; - -// // Now ensure all results are the same. -// REQUIRE(layer1.InputWidth() == layer2.InputWidth()); -// REQUIRE(layer1.InputHeight() == layer2.InputHeight()); -// REQUIRE(layer1.KernelWidth() == layer2.KernelWidth()); -// REQUIRE(layer1.KernelHeight() == layer2.KernelHeight()); -// REQUIRE(layer1.StrideWidth() == layer2.StrideWidth()); -// REQUIRE(layer1.StrideHeight() == layer2.StrideHeight()); -// REQUIRE(layer1.Padding().PadHTop() == layer2.Padding().PadHTop()); -// REQUIRE(layer1.Padding().PadHBottom() == -// layer2.Padding().PadHBottom()); -// REQUIRE(layer1.Padding().PadWLeft() == -// layer2.Padding().PadWLeft()); -// REQUIRE(layer1.Padding().PadWRight() == -// layer2.Padding().PadWRight()); -// REQUIRE(layer1.DilationWidth() == layer2.DilationWidth()); -// REQUIRE(layer1.DilationHeight() == layer2.DilationHeight()); -// } - -// /** -// * Test that the padding options are working correctly in Atrous Convolution -// * layer. -// */ -// TEST_CASE("AtrousConvolutionLayerPaddingTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta; - -// // Check valid padding option. -// AtrousConvolution<> module1(1, 1, 3, 3, 1, 1, -// std::tuple(1, 1), std::tuple(1, 1), 7, 7, -// 2, 2, "valid"); - -// // Test the Forward function. -// input = arma::linspace(0, 48, 49); -// module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); -// module1.Reset(); -// module1.Forward(input, output); - -// REQUIRE(arma::accu(output) == 0); -// REQUIRE(output.n_rows == 9); -// REQUIRE(output.n_cols == 1); - -// // Test the Backward function. -// module1.Backward(input, output, delta); - -// // Check same padding option. -// AtrousConvolution<> module2(1, 1, 3, 3, 1, 1, -// std::tuple(0, 0), std::tuple(0, 0), 7, 7, -// 2, 2, "same"); - -// // Test the forward function. -// input = arma::linspace(0, 48, 49); -// module2.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); -// module2.Reset(); -// module2.Forward(input, output); - -// REQUIRE(arma::accu(output) == 0); -// REQUIRE(output.n_rows == 49); -// REQUIRE(output.n_cols == 1); - -// // Test the backward function. -// module2.Backward(input, output, delta); -// } - -/** - * Tests the GroupNorm layer. - */ -// TEST_CASE("GroupNormTest", "[ANNLayerTest]") -// { -// arma::mat input, output, backwardOutput; -// input = { -// { 2, 0, 1 }, -// { 3, 1, 2 }, -// { 5, 1, 3 }, -// { 7, 2, 4 }, -// { 11, 3, 5 }, -// { 13, 5, 6 }, -// { 17, 8, 7 }, -// { 19, 13, 8 } -// }; -// -// GroupNorm<> model(2, 4); -// model.Reset(); -// -// model.Forward(input, output); -// arma::mat result; -// result = { -// { -1.1717001972, -1.4142135482, -1.3416407811 }, -// { -0.6509445540, 0.0000000000 , -0.4472135937 }, -// { 0.3905667324 , 0.0000000000 , 0.4472135937 }, -// { 1.4320780188 , 1.4142135482 , 1.341640781 }, -// { -1.2649110634, -1.1283296293, -1.3416407811 }, -// { -0.6324555317, -0.5973509802, -0.4472135937 }, -// { 0.6324555317 , 0.1991169934 , 0.4472135937 }, -// { 1.2649110634 , 1.5265636161 , 1.3416407811 } -// }; -// -// CheckMatrices(output, result, 1e-5); -// } - -/** - * GroupNorm layer numerical gradient test. - */ -// TEST_CASE("GradientGroupNormTest", "[ANNLayerTest]") -// { -// // Add function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randn(10, 256)), -// target(arma::zeros(1, 256)) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(10, 10); -// model->Add >(1, 10); -// model->Add >(10, 2); -// model->Add >(); -// } -// -// ~GradientFunction() -// { -// delete model; -// } -// -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 256, false); -// model->Gradient(model->Parameters(), 0, gradient, 256); -// return error; -// } -// -// arma::mat& Parameters() { return model->Parameters(); } -// -// FFN* model; -// arma::mat input, target; -// } function; -// -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -/** - * Tests the LayerNorm layer. - * -TEST_CASE("LayerNormTest", "[ANNLayerTest]") -{ - arma::mat input, output; - input = { { 5.1, 3.5 }, - { 4.9, 3.0 }, - { 4.7, 3.2 } }; - - LayerNorm model(input.n_rows); - model.Reset(); - - model.Forward(input, output); - arma::mat result; - result = { { 1.2247, 1.2978 }, - { 0, -1.1355 }, - { -1.2247, -0.1622 } }; - - CheckMatrices(output, result, 1e-1); - result.clear(); - - output = model.Mean(); - result = { 4.9000, 3.2333 }; - - CheckMatrices(output, result, 1e-1); - result.clear(); - - output = model.Variance(); - result = { 0.0267, 0.0422 }; - - CheckMatrices(output, result, 1e-1); -} -*/ - -/** - * LayerNorm layer numerical gradient test. - * -TEST_CASE("GradientLayerNormTest", "[ANNLayerTest]") -{ - // Add function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - input(arma::randn(10, 256)), - target(arma::zeros(1, 256)) - { - model = new FFN(); - model->ResetData(input, target); - model->Add(); - model->Add(10, 10); - model->Add(10); - model->Add(10, 2); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 16, false); - model->Gradient(model->Parameters(), 0, gradient, 16); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - REQUIRE(CheckGradient(function) <= 1e-4); -} -*/ - -/** - * Test that the functions that can access the parameters of the - * Layer Norm layer work. - * -TEST_CASE("LayerNormLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : size, eps. - LayerNorm layer(5, 1e-3); - - // Make sure we can get the parameters successfully. - REQUIRE(layer.InSize() == 5); - REQUIRE(layer.Epsilon() == 1e-3); -} -*/ - -// /** -// * Test if the AddMerge layer is able to forward the -// * Forward/Backward/Gradient calls. -// */ -// TEST_CASE("AddMergeRunTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta, error; - -// AddMerge<> module(true, true); - -// Linear<>* linear = new Linear<>(10, 10); -// module.Add(linear); - -// linear->Parameters().randu(); -// linear->Reset(); - -// input = arma::zeros(10, 1); -// module.Forward(input, output); - -// double parameterSum = arma::accu(linear->Parameters().submat( -// 100, 0, linear->Parameters().n_elem - 1, 0)); - -// // Test the Backward function. -// module.Backward(input, input, delta); - -// // Clean up before we break, -// delete linear; - -// REQUIRE(parameterSum == Approx(arma::accu(output)).epsilon(1e-5)); -// REQUIRE(arma::accu(delta) == 0); -// } - -/** - * Test if the MultiplyMerge layer is able to forward the - * Forward/Backward/Gradient calls. - * -TEST_CASE("MultiplyMergeRunTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta, error; - - MultiplyMerge module(true, true); - - Linear* linear = new Linear(10, 10); - module.Add(linear); - - linear->Parameters().randu(); - linear->Reset(); - - input = arma::zeros(10, 1); - module.Forward(input, output); - - double parameterSum = arma::accu(linear->Parameters().submat( - 100, 0, linear->Parameters().n_elem - 1, 0)); - - // Test the Backward function. - module.Backward(input, input, delta); - - // Clean up before we break, - delete linear; - - REQUIRE(parameterSum == Approx(arma::accu(output)).epsilon(1e-5)); - REQUIRE(arma::accu(delta) == 0); -} -*/ - -/** - * Simple subview module test. - * -TEST_CASE("SimpleSubviewLayerTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta, outputMat; - Subview moduleRow(1, 10, 19); - - // Test the Forward function for a vector. - input = arma::ones(20, 1); - moduleRow.Forward(input, output); - REQUIRE(output.n_rows == 10); - - Subview moduleMat(4, 3, 6, 0, 2); - - // Test the Forward function for a matrix. - input = arma::ones(20, 8); - moduleMat.Forward(input, outputMat); - REQUIRE(outputMat.n_rows == 12); - REQUIRE(outputMat.n_cols == 2); - - // Test the Backward function. - moduleMat.Backward(input, input, delta); - REQUIRE(accu(delta) == 160); - REQUIRE(delta.n_rows == 20); -} -*/ - -/** - * Subview index test. - * -TEST_CASE("SubviewIndexTest", "[ANNLayerTest]") -{ - arma::mat outputEnd, outputMid, outputStart, input, delta; - input = arma::linspace(1, 20, 20); - - // Slicing from the initial indices. - Subview moduleStart(1, 0, 9); - arma::mat subStart = arma::linspace(1, 10, 10); - - moduleStart.Forward(input, outputStart); - CheckMatrices(outputStart, subStart); - - // Slicing from the mid indices. - Subview moduleMid(1, 6, 15); - arma::mat subMid = arma::linspace(7, 16, 10); - - moduleMid.Forward(input, outputMid); - CheckMatrices(outputMid, subMid); - - // Slicing from the end indices. - Subview moduleEnd(1, 10, 19); - arma::mat subEnd = arma::linspace(11, 20, 10); - - moduleEnd.Forward(input, outputEnd); - CheckMatrices(outputEnd, subEnd); -} -*/ - -/** - * Subview batch test. - * -TEST_CASE("SubviewBatchTest", "[ANNLayerTest]") -{ - arma::mat output, input, outputCol, outputMat, outputDef; - - // All rows selected. - Subview moduleCol(1, 0, 19); - - // Test with inSize 1. - input = arma::ones(20, 8); - moduleCol.Forward(input, outputCol); - CheckMatrices(outputCol, input); - - // Few rows and columns selected. - Subview moduleMat(4, 3, 6, 0, 2); - - // Test with inSize greater than 1. - moduleMat.Forward(input, outputMat); - output = arma::ones(12, 2); - CheckMatrices(outputMat, output); - - // endCol changed to 3 by default. - Subview moduleDef(4, 1, 6, 0, 4); - - // Test with inSize greater than 1 and endCol >= inSize. - moduleDef.Forward(input, outputDef); - output = arma::ones(24, 2); - CheckMatrices(outputDef, output); -} -*/ - -/** - * Test that the functions that can modify and access the parameters of the - * Subview layer work. - * -TEST_CASE("SubviewLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : inSize, beginRow, endRow, beginCol, endCol. - Subview layer1(1, 2, 3, 4, 5); - Subview layer2(1, 3, 4, 5, 6); - - // Make sure we can get the parameters correctly. - REQUIRE(layer1.InSize() == 1); - REQUIRE(layer1.BeginRow() == 2); - REQUIRE(layer1.EndRow() == 3); - REQUIRE(layer1.BeginCol() == 4); - REQUIRE(layer1.EndCol() == 5); - - // Now modify the parameters to match the second layer. - layer1.BeginRow() = 3; - layer1.EndRow() = 4; - layer1.BeginCol() = 5; - layer1.EndCol() = 6; - - // Now ensure all results are the same. - REQUIRE(layer1.InSize() == layer2.InSize()); - REQUIRE(layer1.BeginRow() == layer2.BeginRow()); - REQUIRE(layer1.EndRow() == layer2.EndRow()); - REQUIRE(layer1.BeginCol() == layer2.BeginCol()); - REQUIRE(layer1.EndCol() == layer2.EndCol()); -} -*/ - -/* - * Simple Reparametrization module test. - * -TEST_CASE("SimpleReparametrizationLayerTest", "[ANNLayerTest]") -{ - arma::mat input, output, delta; - Reparametrization module(5); - - // Test the Forward function. As the mean is zero and the standard - // deviation is small, after multiplying the gaussian sample, the - // output should be small enough. - input = join_cols(arma::ones(5, 1) * -15, - arma::zeros(5, 1)); - module.Forward(input, output); - REQUIRE(arma::accu(output) <= 1e-5); - - // Test the Backward function. - arma::mat gy = arma::zeros(5, 1); - module.Backward(input, gy, delta); - REQUIRE(arma::accu(delta) != 0); // klBackward will be added. -} -*/ - -/** - * Reparametrization module stochastic boolean test. - * -TEST_CASE("ReparametrizationLayerStochasticTest", "[ANNLayerTest]") -{ - arma::mat input, outputA, outputB; - Reparametrization module(5, false); - - input = join_cols(arma::ones(5, 1), - arma::zeros(5, 1)); - - // Test if two forward passes generate same output. - module.Forward(input, outputA); - module.Forward(input, outputB); - - CheckMatrices(outputA, outputB); -} -*/ - -/** - * Reparametrization module includeKl boolean test. - * -TEST_CASE("ReparametrizationLayerIncludeKlTest", "[ANNLayerTest]") -{ - arma::mat input, output, gy, delta; - Reparametrization module(5, true, false); - - input = join_cols(arma::ones(5, 1), - arma::zeros(5, 1)); - module.Forward(input, output); - - // As KL divergence is not included, with the above inputs, the delta - // matrix should be all zeros. - gy = arma::zeros(output.n_rows, output.n_cols); - module.Backward(output, gy, delta); - - REQUIRE(arma::accu(delta) == 0); -} -*/ - -/** - * Jacobian Reparametrization module test. - * -TEST_CASE("JacobianReparametrizationLayerTest", "[ANNLayerTest]") -{ - for (size_t i = 0; i < 5; ++i) - { - const size_t inputElementsHalf = math::RandInt(2, 10); - - arma::mat input; - input.set_size(inputElementsHalf * 2, 1); - - Reparametrization module(inputElementsHalf, false, false); - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -} -*/ - -/** - * Reparametrization layer numerical gradient test. - * -TEST_CASE("GradientReparametrizationLayerTest", "[ANNLayerTest]") -{ - // Linear function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - input(arma::randu(10, 1)), - target(arma::mat("0")) - { - model = new FFN(); - model->ResetData(input, target); - model->Add(); - model->Add(10, 6); - model->Add(3, false, true, 1); - model->Add(3, 2); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - // REQUIRE(CheckGradient(function) <= 1e-4); -} -*/ - -/** - * Reparametrization layer beta numerical gradient test. - * -TEST_CASE("GradientReparametrizationLayerBetaTest", "[ANNLayerTest]") -{ - // Linear function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - input(arma::randu(10, 2)), - target(arma::mat("0 0")) - { - model = new FFN(); - model->ResetData(input, target); - model->Add(); - model->Add(10, 6); - // Use a value of beta not equal to 1. - model->Add(3, false, true, 2); - model->Add(3, 2); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - // REQUIRE(CheckGradient(function) <= 1e-4); -} -*/ - -/** - * Test that the functions that can access the parameters of the - * Reparametrization layer work. - * -TEST_CASE("ReparametrizationLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : latentSize, stochastic, includeKL, beta. - Reparametrization layer(5, false, false, 2); - - // Make sure we can get the parameters successfully. - REQUIRE(layer.OutputSize() == 5); - REQUIRE(layer.Stochastic() == false); - REQUIRE(layer.IncludeKL() == false); - REQUIRE(layer.Beta() == 2); -} -*/ - -/** - * Simple residual module test. - * -TEST_CASE("SimpleResidualLayerTest", "[ANNLayerTest]") -{ - arma::mat outputA, outputB, input, deltaA, deltaB; - - Sequential* sequential = new Sequential(true); - Residual* residual = new Residual(true); - - Linear* linearA = new Linear(10, 10); - linearA->Parameters().randu(); - linearA->Reset(); - Linear* linearB = new Linear(10, 10); - linearB->Parameters().randu(); - linearB->Reset(); - - // Add the same layers (with the same parameters) to both Sequential and - // Residual object. - sequential->Add(linearA); - sequential->Add(linearB); - - residual->Add(linearA); - residual->Add(linearB); - - // Test the Forward function (pass the same input to both). - input = arma::randu(10, 1); - sequential->Forward(input, outputA); - residual->Forward(input, outputB); - - CheckMatrices(outputA, outputB - input); - - // Test the Backward function (pass the same error to both). - sequential->Backward(input, input, deltaA); - residual->Backward(input, input, deltaB); - - CheckMatrices(deltaA, deltaB - input); - - delete sequential; - delete residual; - delete linearA; - delete linearB; -} -*/ - -/** - * Simple Highway module test. - * -TEST_CASE("SimpleHighwayLayerTest", "[ANNLayerTest]") -{ - arma::mat outputA, outputB, input, deltaA, deltaB; - Sequential* sequential = new Sequential(true); - Highway* highway = new Highway(10, true); - highway->Parameters().zeros(); - highway->Reset(); - - Linear* linearA = new Linear(10, 10); - linearA->Parameters().randu(); - linearA->Reset(); - Linear* linearB = new Linear(10, 10); - linearB->Parameters().randu(); - linearB->Reset(); - - // Add the same layers (with the same parameters) to both Sequential and - // Highway object. - highway->Add(linearA); - highway->Add(linearB); - sequential->Add(linearA); - sequential->Add(linearB); - - // Test the Forward function (pass the same input to both). - input = arma::randu(10, 1); - sequential->Forward(input, outputA); - highway->Forward(input, outputB); - - CheckMatrices(outputB, input * 0.5 + outputA * 0.5); - - delete sequential; - delete highway; - delete linearA; - delete linearB; -} -*/ - -/** - * Test that the function that can access the inSize parameter of the - * Highway layer works. - * -TEST_CASE("HighwayLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : inSize, model. - Highway layer(1, true); - - // Make sure we can get the parameter successfully. - REQUIRE(layer.InSize() == 1); -} -*/ - -// /** -// * Sequential layer numerical gradient test. -// */ -// TEST_CASE("GradientHighwayLayerTest", "[ANNLayerTest]") -// { -// // Linear function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(5, 1)), -// target(arma::mat("0")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add(); -// model->Add(5, 10); - -// highway = new Highway(10); -// highway->Add(10, 10); -// highway->Add(); -// highway->Add(10, 10); -// highway->Add(); - -// model->Add(highway); -// model->Add(10, 2); -// model->Add(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// Highway* highway; -// arma::mat input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -/** - * Sequential layer numerical gradient test. - */ -// TEST_CASE("GradientSequentialLayerTest", "[ANNLayerTest]") -// { -// // Linear function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(10, 1)), -// target(arma::mat("0")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add(); -// model->Add(10, 10); -// sequential = new Sequential(); -// sequential->Add(10, 10); -// sequential->Add(); -// sequential->Add(10, 5); -// sequential->Add(); - -// model->Add(sequential); -// model->Add(5, 2); -// model->Add(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// Sequential* sequential; -// arma::mat input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -// /** -// * WeightNorm layer numerical gradient test. -// */ -// TEST_CASE("GradientWeightNormLayerTest", "[ANNLayerTest]") -// { -// // Linear function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(10, 1)), -// target(arma::mat("0")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add(10, 10); - -// Linear* linear = new Linear(10, 2); -// weightNorm = new WeightNorm(linear); - -// model->Add(weightNorm); -// model->Add(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// WeightNorm* weightNorm; -// arma::mat input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -// /** -// * Test if the WeightNorm layer is able to forward the -// * Forward/Backward/Gradient calls. -// */ -// TEST_CASE("WeightNormRunTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta, error; -// Linear* linear = new Linear(10, 10); - -// WeightNorm module(linear); - -// module.Parameters().randu(); -// module.Reset(); - -// linear->Bias().zeros(); - -// input = arma::zeros(10, 1); -// module.Forward(input, output); - -// // Test the Backward function. -// module.Backward(input, input, delta); - -// REQUIRE(0 == arma::accu(output)); -// REQUIRE(arma::accu(delta) == 0); -// } - -// // General ANN serialization test. -// template -// void ANNLayerSerializationTest(LayerType& layer) -// { -// arma::mat input(5, 100, arma::fill::randu); -// arma::mat output(5, 100, arma::fill::randu); - -// FFN model; -// model.Add>(input.n_rows, 10); -// model.Add(layer); -// model.Add>(); -// model.Add>(10, output.n_rows); -// model.Add>(); - -// ens::StandardSGD opt(0.1, 1, 5, -100, false); -// model.Train(input, output, opt); - -// arma::mat originalOutput; -// model.Predict(input, originalOutput); - -// // Now serialize the model. -// FFN xmlModel, jsonModel, -// binaryModel; -// SerializeObjectAll(model, xmlModel, jsonModel, binaryModel); - -// // Ensure that predictions are the same. -// arma::mat modelOutput, xmlOutput, jsonOutput, binaryOutput; -// model.Predict(input, modelOutput); -// xmlModel.Predict(input, xmlOutput); -// jsonModel.Predict(input, jsonOutput); -// binaryModel.Predict(input, binaryOutput); - -// CheckMatrices(originalOutput, modelOutput, 1e-5); -// CheckMatrices(originalOutput, xmlOutput, 1e-5); -// CheckMatrices(originalOutput, jsonOutput, 1e-5); -// CheckMatrices(originalOutput, binaryOutput, 1e-5); -// } - -// /** -// * Simple serialization test for batch normalization layer. -// */ -// TEST_CASE("BatchNormSerializationTest", "[ANNLayerTest]") -// { -// BatchNorm<> layer(10); -// ANNLayerSerializationTest(layer); -// } - -// /** -// * Simple serialization test for layer normalization layer. -// */ -// TEST_CASE("LayerNormSerializationTest", "[ANNLayerTest]") -// { -// LayerNorm<> layer(10); -// ANNLayerSerializationTest(layer); -// } - /** * Test that the functions that can modify and access the parameters of the * Convolution layer work. @@ -4707,205 +630,6 @@ TEST_CASE("GradientConvolutionLayerTest", "[ANNLayerTest]") REQUIRE(CheckGradient(function) < 1e3); } -/** - * Test that the padding options in Transposed Convolution layer. - * -TEST_CASE("TransposedConvolutionLayerPaddingTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta; - - TransposedConvolution module1(1, 1, 3, 3, 1, 1, 0, 0, 4, 4, 6, 6, "VALID"); - // Test the forward function. - // Valid Should give the same result. - input = arma::linspace(0, 15, 16); - module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); - module1.Reset(); - module1.Forward(input, output); - // Value calculated using tensorflow.nn.conv2d_transpose(). - REQUIRE(arma::accu(output) == 0.0); - - // Test the Backward Function. - module1.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 0.0); - - // Test Valid for non zero padding. - TransposedConvolution module2(1, 1, 3, 3, 2, 2, - std::tuple(0, 0), std::tuple(0, 0), - 2, 2, 5, 5, "VALID"); - // Test the forward function. - input = arma::linspace(0, 3, 4); - module2.Parameters() = arma::mat(25 + 1, 1, arma::fill::zeros); - module2.Parameters()(2) = 8.0; - module2.Parameters()(4) = 6.0; - module2.Parameters()(6) = 4.0; - module2.Parameters()(8) = 2.0; - module2.Reset(); - module2.Forward(input, output); - // Value calculated using torch.nn.functional.conv_transpose2d(). - REQUIRE(arma::accu(output) == 120.0); - - // Test the Backward Function. - module2.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 960.0); - - // Test for same padding type. - TransposedConvolution module3(1, 1, 3, 3, 2, 2, 0, 0, 3, 3, 3, 3, "SAME"); - // Test the forward function. - input = arma::linspace(0, 8, 9); - module3.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); - module3.Reset(); - module3.Forward(input, output); - REQUIRE(arma::accu(output) == 0); - REQUIRE(output.n_rows == input.n_rows); - REQUIRE(output.n_cols == input.n_cols); - - // Test the Backward Function. - module3.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 0.0); - - // Output shape should equal input. - TransposedConvolution module4(1, 1, 3, 3, 1, 1, - std::tuple(2, 2), std::tuple(2, 2), - 5, 5, 5, 5, "SAME"); - // Test the forward function. - input = arma::linspace(0, 24, 25); - module4.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); - module4.Reset(); - module4.Forward(input, output); - REQUIRE(arma::accu(output) == 0); - REQUIRE(output.n_rows == input.n_rows); - REQUIRE(output.n_cols == input.n_cols); - - // Test the Backward Function. - module4.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 0.0); - - TransposedConvolution module5(1, 1, 3, 3, 2, 2, 0, 0, 2, 2, 2, 2, "SAME"); - // Test the forward function. - input = arma::linspace(0, 3, 4); - module5.Parameters() = arma::mat(25 + 1, 1, arma::fill::zeros); - module5.Reset(); - module5.Forward(input, output); - REQUIRE(arma::accu(output) == 0); - REQUIRE(output.n_rows == input.n_rows); - REQUIRE(output.n_cols == input.n_cols); - - // Test the Backward Function. - module5.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 0.0); - - TransposedConvolution module6(1, 1, 4, 4, 1, 1, 1, 1, 5, 5, 5, 5, "SAME"); - // Test the forward function. - input = arma::linspace(0, 24, 25); - module6.Parameters() = arma::mat(16 + 1, 1, arma::fill::zeros); - module6.Reset(); - module6.Forward(input, output); - REQUIRE(arma::accu(output) == 0); - REQUIRE(output.n_rows == input.n_rows); - REQUIRE(output.n_cols == input.n_cols); - - // Test the Backward Function. - module6.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 0.0); -} -*/ - -/** - * Simple test for Lp Pooling layer. - */ -// TEST_CASE("LpMaxPoolingTestCase", "[ANNLayerTest]") -// { -// // For rectangular input to pooling layers. -// arma::mat input = arma::mat(8, 1); -// arma::mat output; -// input.zeros(); -// input(0) = input(6) = 30; -// input(1) = input(7) = 120; -// input(2) = input(4) = 272; -// input(3) = input(5) = 315; -// // Output-Size should be 1 x 2. -// // Square output. -// LpPooling<> module1(4, 2, 2, 2, 2); -// module1.InputHeight() = 2; -// module1.InputWidth() = 4; -// module1.Forward(input, output); -// // Calculated using torch.nn.LPPool2d(). -// REQUIRE(arma::accu(output) - 706.0 == Approx(0.0).margin(2e-5)); -// REQUIRE(output.n_elem == 2); -// -// // For Square input. -// input = arma::mat(16, 1); -// input.zeros(); -// input(0) = 4; -// input(1) = 3; -// input(3) = 12; -// input(7) = 35; -// input(8) = 6; -// input(11) = 7; -// input(12) = 8; -// input(15) = 24; -// // Output-Size should be 2 x 2. -// // Square output. -// LpPooling<> module3(2, 2, 2, 2, 2); -// module3.InputHeight() = 4; -// module3.InputWidth() = 4; -// module3.Forward(input, output); -// // Calculated using torch.nn.LPPool2d(). -// REQUIRE(arma::accu(output) - 77.0 == Approx(0.0).margin(2e-5)); -// REQUIRE(output.n_elem == 4); -// } - -/** - * Simple test for Mean Pooling layer. - */ -// TEST_CASE("MeanPoolingTestCase", "[ANNLayerTest]") -// { -// // For rectangular input to pooling layers. -// arma::mat input = arma::mat(28, 1); -// input.zeros(); -// input(0) = input(16) = 1; -// input(1) = input(17) = 2; -// input(2) = input(18) = 3; -// input(3) = input(19) = 4; -// input(4) = input(20) = 5; -// input(5) = input(23) = 6; -// input(6) = input(24) = 7; -// input(14) = input(25) = 8; -// input(15) = input(26) = 9; -// -// MeanPooling<> module1(2, 2, 2, 2, false); -// MeanPooling<> module2(2, 2, 2, 2, true); -// module1.InputWidth() = 7; -// module1.InputHeight() = 4; -// module2.InputWidth() = 7; -// module2.InputHeight() = 4; -// -// // Calculated using torch.nn.MeanPool2d(). -// arma::mat result1, result2; -// result1 << 0.7500 << 4.2500 << arma::endr -// << 1.7500 << 4.0000 << arma::endr -// << 2.7500 << 6.0000 << arma::endr -// << 3.5000 << 2.5000 << arma::endr; -// -// result2 << 0.7500 << 4.2500 << arma::endr -// << 1.7500 << 4.0000 << arma::endr -// << 2.7500 << 6.0000 << arma::endr; -// -// arma::mat output1, output2; -// module1.Forward(input, output1); -// module2.Forward(input, output2); -// output1.reshape(4, 2); -// output2.reshape(3, 2); -// CheckMatrices(output1, result1, 1e-1); -// CheckMatrices(output2, result2, 1e-1); -// -// arma::mat delta1, delta2; -// module1.Backward(input, output1, delta1); -// REQUIRE(arma::accu(delta1) == 25.5); -// module2.Backward(input, output2, delta2); -// REQUIRE(arma::accu(delta2) == 19.5); -// } - /** * Simple test for Max Pooling layer. */ @@ -4999,469 +723,6 @@ TEST_CASE("MaxPoolingTestCase", "[ANNLayerTest]") REQUIRE(output.n_cols == 1); } -/** - * Test that the functions that can modify and access the parameters of the - * Glimpse layer work. - * -TEST_CASE("GlimpseLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : inSize, size, depth, scale, inputWidth, inputHeight. - Glimpse layer1(1, 2, 3, 4, 5, 6); - Glimpse layer2(1, 2, 3, 4, 6, 7); - - // Make sure we can get the parameters successfully. - REQUIRE(layer1.InputHeight() == 6); - REQUIRE(layer1.InputWidth() == 5); - REQUIRE(layer1.Scale() == 4); - REQUIRE(layer1.Depth() == 3); - REQUIRE(layer1.GlimpseSize() == 2); - REQUIRE(layer1.InSize() == 1); - - // Now modify the parameters to match the second layer. - layer1.InputHeight() = 7; - layer1.InputWidth() = 6; - - // Now ensure that all the results are the same. - REQUIRE(layer1.InputHeight() == layer2.InputHeight()); - REQUIRE(layer1.InputWidth() == layer2.InputWidth()); - REQUIRE(layer1.Scale() == layer2.Scale()); - REQUIRE(layer1.Depth() == layer2.Depth()); - REQUIRE(layer1.GlimpseSize() == layer2.GlimpseSize()); - REQUIRE(layer1.InSize() == layer2.InSize()); -} -*/ - -/** - * Test that the function that can access the stdev parameter of the - * Reinforce Normal layer works. - * -TEST_CASE("ReinforceNormalLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter : stdev. - ReinforceNormal layer(4.0); - - // Make sure we can get the parameter successfully. - REQUIRE(layer.StandardDeviation() == 4.0); -} -*/ - -/** - * Simple test for Adaptive pooling for Max Pooling layer. - * -TEST_CASE("AdaptiveMaxPoolingTestCase", "[ANNLayerTest]") -{ - // For rectangular input. - arma::mat input = arma::mat(12, 1); - arma::mat output, delta; - - input.zeros(); - input(0) = 1; - input(1) = 2; - input(2) = 3; - input(3) = input(8) = 7; - input(4) = 4; - input(5) = 5; - input(6) = input(7) = 6; - input(10) = 8; - input(11) = 9; - // Output-Size should be 2 x 2. - // Square output. - AdaptiveMaxPooling module1(2, 2); - module1.InputHeight() = 3; - module1.InputWidth() = 4; - module1.Forward(input, output); - // Calculated using torch.nn.AdaptiveMaxPool2d(). - REQUIRE(arma::accu(output) == 28); - REQUIRE(output.n_elem == 4); - REQUIRE(output.n_cols == 1); - // Test the Backward Function. - module1.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 28.0); - - // For Square input. - input = arma::mat(9, 1); - input.zeros(); - input(0) = 6; - input(1) = 3; - input(2) = 9; - input(3) = 3; - input(6) = 3; - // Output-Size should be 1 x 2. - // Rectangular output. - AdaptiveMaxPooling module2(2, 1); - module2.InputHeight() = 3; - module2.InputWidth() = 3; - module2.Forward(input, output); - // Calculated using torch.nn.AdaptiveMaxPool2d(). - REQUIRE(arma::accu(output) == 15.0); - REQUIRE(output.n_elem == 2); - REQUIRE(output.n_cols == 1); - // Test the Backward Function. - module2.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 15.0); - - // For Square input. - input = arma::mat(16, 1); - input.zeros(); - input(0) = 6; - input(1) = 3; - input(2) = 9; - input(4) = 3; - input(8) = 3; - // Output-Size should be 3 x 3. - // Square output. - AdaptiveMaxPooling module3(std::tuple(3, 3)); - module3.InputHeight() = 4; - module3.InputWidth() = 4; - module3.Forward(input, output); - // Calculated using torch.nn.AdaptiveMaxPool2d(). - REQUIRE(arma::accu(output) == 30.0); - REQUIRE(output.n_elem == 9); - REQUIRE(output.n_cols == 1); - // Test the Backward Function. - module3.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 30.0); - - // For Rectangular input. - input = arma::mat(20, 1); - input.zeros(); - input(0) = 1; - input(1) = 1; - input(3) = 1; - // Output-Size should be 2 x 2. - // Square output. - AdaptiveMaxPooling module4(std::tuple(2, 2)); - module4.InputHeight() = 4; - module4.InputWidth() = 5; - module4.Forward(input, output); - // Calculated using torch.nn.AdaptiveMaxPool2d(). - REQUIRE(arma::accu(output) == 2); - REQUIRE(output.n_elem == 4); - REQUIRE(output.n_cols == 1); - // Test the Backward Function. - module4.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 2.0); -} -*/ - -/** - * Simple test for Adaptive pooling for Mean Pooling layer. - * -TEST_CASE("AdaptiveMeanPoolingTestCase", "[ANNLayerTest]") -{ - // For rectangular input. - arma::mat input = arma::mat(12, 1); - arma::mat output, delta; - - input.zeros(); - input(0) = 1; - input(1) = 2; - input(2) = 3; - input(3) = input(8) = 7; - input(4) = 4; - input(5) = 5; - input(6) = input(7) = 6; - input(10) = 8; - input(11) = 9; - // Output-Size should be 2 x 2. - // Square output. - AdaptiveMeanPooling module1(2, 2); - module1.InputHeight() = 3; - module1.InputWidth() = 4; - module1.Forward(input, output); - // Calculated using torch.nn.AdaptiveAvgPool2d(). - REQUIRE(arma::accu(output) == 19.75); - REQUIRE(output.n_elem == 4); - REQUIRE(output.n_cols == 1); - // Test the Backward Function. - module1.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 19.75); - - // For Square input. - input = arma::mat(9, 1); - input.zeros(); - input(0) = 6; - input(1) = 3; - input(2) = 9; - input(3) = 3; - input(6) = 3; - // Output-Size should be 1 x 2. - // Rectangular output. - AdaptiveMeanPooling module2(1, 2); - module2.InputHeight() = 3; - module2.InputWidth() = 3; - module2.Forward(input, output); - // Calculated using torch.nn.AdaptiveAvgPool2d(). - REQUIRE(arma::accu(output) == 4.5); - REQUIRE(output.n_elem == 2); - REQUIRE(output.n_cols == 1); - // Test the Backward Function. - module2.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 4.50); - - // For Square input. - input = arma::mat(16, 1); - input.zeros(); - input(0) = 6; - input(1) = 3; - input(2) = 9; - input(4) = 3; - input(8) = 3; - // Output-Size should be 3 x 3. - // Square output. - AdaptiveMeanPooling module3(std::tuple(3, 3)); - module3.InputHeight() = 4; - module3.InputWidth() = 4; - module3.Forward(input, output); - // Calculated using torch.nn.AdaptiveAvgPool2d(). - REQUIRE(arma::accu(output) == 10.5); - REQUIRE(output.n_elem == 9); - REQUIRE(output.n_cols == 1); - // Test the Backward Function. - module3.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 10.5); - - // For Rectangular input. - input = arma::mat(24, 1); - input.zeros(); - input(0) = 3; - input(1) = 3; - input(4) = 3; - // Output-Size should be 3 x 3. - // Square output. - AdaptiveMeanPooling module4(std::tuple(3, 3)); - module4.InputHeight() = 4; - module4.InputWidth() = 6; - module4.Forward(input, output); - // Calculated using torch.nn.AdaptiveAvgPool2d(). - REQUIRE(arma::accu(output) == 2.25); - REQUIRE(output.n_elem == 9); - REQUIRE(output.n_cols == 1); - // Test the Backward Function. - module4.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 2.25); -} -*/ - -/* -TEST_CASE("TransposedConvolutionalLayerOptionalParameterTest", "[ANNLayerTest]") -{ - Sequential* decoder = new Sequential(); - - // Check if we can create an object without specifying output. - REQUIRE_NOTHROW(decoder->Add(24, 16, - 5, 5, 1, 1, 0, 0, 10, 10)); - - REQUIRE_NOTHROW(decoder->Add(16, 1, - 15, 15, 1, 1, 1, 1, 14, 14)); - - delete decoder; -} -*/ - -// TEST_CASE("BatchNormWithMinBatchesTest", "[ANNLayerTest]") -// { -// arma::mat input, output, result, runningMean, runningVar, delta; - -// // The input test matrix is of the form 3 x 2 x 4 x 1 where -// // number of images are 3 and number of feature maps are 2. -// input = { { 1, 446, 42 }, -// { 2, 16, 63 }, -// { 3, 13, 63 }, -// { 4, 21, 21 }, -// { 1, 13, 11 }, -// { 32, 45, 42 }, -// { 22, 16, 63 }, -// { 32, 13, 42 } }; -// -// // Output calculated using torch.nn.BatchNorm2d(). -// result = { { -0.4786, 3.2634, -0.1338 }, -// { -0.4702, -0.3525, 0.0427 }, -// { -0.4618, -0.3777, 0.0427 }, -// { -0.4534, -0.3104, -0.3104 }, -// { -1.5429, -0.8486, -0.9643 }, -// { 0.2507, 1.0029, 0.8293 }, -// { -0.3279, -0.675, 2.0443 }, -// { 0.2507 , -0.8486 , 0.8293 } }; - -// // Check correctness of batch normalization. -// BatchNorm<> module1(2, 1e-5, false, 0.1); -// module1.Reset(); -// module1.Forward(input, output); -// CheckMatrices(output, result, 1e-1); - -// // Check backward function. -// module1.Backward(input, output, delta); -// REQUIRE(arma::accu(delta) == Approx(0.0102676).epsilon(1e-5)); - -// // Check values for running mean and running variance. -// // Calculated using torch.nn.BatchNorm2d(). -// runningMean = arma::mat(2, 1); -// runningVar = arma::mat(2, 1); -// runningMean(0) = 5.7917; -// runningMean(1) = 2.76667; -// runningVar(0) = 1543.6545; -// runningVar(1) = 33.488; - -// CheckMatrices(runningMean, module1.TrainingMean(), 1e-3); -// CheckMatrices(runningVar, module1.TrainingVariance(), 1e-2); - -// // Check correctness of layer when running mean and variance -// // are updated using cumulative average. -// BatchNorm<> module2(2); -// module2.Reset(); -// module2.Forward(input, output); -// CheckMatrices(output, result, 1e-1); - -// // Check values for running mean and running variance. -// // Calculated using torch.nn.BatchNorm2d(). -// runningMean(0) = 57.9167; -// runningMean(1) = 27.6667; -// runningVar(0) = 15427.5380; -// runningVar(1) = 325.8787; - -// CheckMatrices(runningMean, module2.TrainingMean(), 1e-2); -// CheckMatrices(runningVar, module2.TrainingVariance(), 1e-2); - -// // Check correctness when model is testing. -// arma::mat deterministicOutput; -// module1.Deterministic() = true; -// module1.Forward(input, deterministicOutput); - -// result.clear(); -// result = { { -0.12195, 11.20426, 0.92158 }, -// { -0.0965, 0.259824, 1.4560 }, -// { -0.071054, 0.183567, 1.45607 }, -// { -0.045601, 0.3870852, 0.38708 }, -// { -0.305288, 1.7683, 1.4227 }, -// { 5.05166, 7.29812, 6.7797 }, -// { 3.323614, 2.2867, 10.4086 }, -// { 5.05166, 1.7683, 6.7797 } }; - -// CheckMatrices(result, deterministicOutput, 1e-1); - -// // Check correctness by updating the running mean and variance again. -// module1.Deterministic() = false; - -// // Clean up. -// output.clear(); -// input.clear(); - -// // The input test matrix is of the form 2 x 2 x 3 x 1 where -// // number of images are 2 and number of feature maps are 2. -// input = { { 12, 443 }, -// { 134, 45 }, -// { 11, 13 }, -// { 14, 55 }, -// { 110, 4 }, -// { 1, 45 } }; -// -// result = { { -0.629337, 2.14791 }, -// { 0.156797, -0.416694 }, -// { -0.63578, -0.622893 }, -// { -0.637481, 0.4440386 }, -// { 1.894857, -0.901267 }, -// { -0.980402, 0.180253 } }; - -// module1.Forward(input, output); -// CheckMatrices(result, output, 1e-3); - -// // Check correctness for the second module as well. -// module2.Forward(input, output); -// CheckMatrices(result, output, 1e-3); - -// // Calculated using torch.nn.BatchNorm2d(). -// runningMean(0) = 16.1792; -// runningMean(1) = 6.30667; -// runningVar(0) = 4276.5849; -// runningVar(1) = 202.595; - -// CheckMatrices(runningMean, module1.TrainingMean(), 1e-3); -// CheckMatrices(runningVar, module1.TrainingVariance(), 1e-1); - -// // Check correctness of running mean and variance when their -// // values are updated using cumulative average. -// runningMean(0) = 83.79166; -// runningMean(1) = 32.9166; -// runningVar(0) = 22164.1035; -// runningVar(1) = 1025.2227; - -// CheckMatrices(runningMean, module2.TrainingMean(), 1e-3); -// CheckMatrices(runningVar, module2.TrainingVariance(), 1e-3); - -// // Check backward function. -// module1.Backward(input, output, delta); - -// deterministicOutput.clear(); -// module1.Deterministic() = true; -// module1.Forward(input, deterministicOutput); - -// result.clear(); -// result = { { -0.06388436, 6.524754114 }, -// { 1.799655281, 0.44047968 }, -// { -0.07913291, -0.04784981 }, -// { 0.5405045, 3.4210097 }, -// { 7.2851023, -0.1620577 }, -// { -0.37282639, 2.7184474 } }; - -// // Calculated using torch.nn.BatchNorm2d(). -// CheckMatrices(result, deterministicOutput, 1e-1); -// } - -// /** -// * Batch Normalization layer numerical gradient test. -// */ -// TEST_CASE("GradientBatchNormWithMiniBatchesTest", "[ANNLayerTest]") -// { -// // Add function gradient instantiation. -// // To make this test robust, check it ten times. -// bool pass = false; -// for (size_t trial = 0; trial < 10; trial++) -// { -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randn(16, 1024)), -// target(arma::zeros(1, 1024)) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add>(); -// model->Add>(1, 2, 3, 3, 1, 1, 0, 0, 4, 4); -// model->Add>(2); -// model->Add>(2 * 2 * 2, 2); -// model->Add>(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1024, false); -// model->Gradient(model->Parameters(), 0, gradient, 1024); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// arma::mat input, target; -// } function; - -// double gradient = CheckGradient(function); -// if (gradient < 1e-1) -// { -// pass = true; -// break; -// } -// } - -// REQUIRE(pass); -// } - TEST_CASE("ConvolutionLayerTestCase", "[ANNLayerTest]") { arma::mat input, output; @@ -5500,78 +761,6 @@ TEST_CASE("ConvolutionLayerTestCase", "[ANNLayerTest]") REQUIRE(arma::accu(output) == 4156); } -// TEST_CASE("BatchNormDeterministicTest", "[ANNLayerTest]") -// { -// FFN<> module; -// module.Add>(2, 1e-5, false); -// module.Add>(); - -// arma::mat input(4, 3), output; -// module.ResetParameters(); - -// // The model should switch to Deterministic mode for predicting. -// module.Predict(input, output); -// REQUIRE(boost::get*>(module.Model()[0])->Deterministic() == true); - -// output.ones(); -// module.Train(input, output); -// // The model should switch to training mode for predicting. -// REQUIRE(boost::get*>(module.Model()[0])->Deterministic() == 0); -// } - -// /** -// * Linear module weight initialization test. -// */ -// TEST_CASE("LinearLayerWeightInitializationTest", "[ANNLayerTest]") -// { -// size_t inSize = 10, outSize = 4; -// Linear<> linear = Linear<>(inSize, outSize); -// linear.Reset(); -// RandomInitialization().Initialize(linear.Weight()); -// linear.Bias().ones(); - -// REQUIRE(std::equal(linear.Weight().begin(), -// linear.Weight().end(), linear.Parameters().begin())); - -// REQUIRE(std::equal(linear.Bias().begin(), -// linear.Bias().end(), linear.Parameters().begin() + inSize * outSize)); - -// REQUIRE(linear.Weight().n_rows == outSize); -// REQUIRE(linear.Weight().n_cols == inSize); -// REQUIRE(linear.Bias().n_rows == outSize); -// REQUIRE(linear.Bias().n_cols == 1); -// REQUIRE(linear.Parameters().n_rows == inSize * outSize + outSize); -// } - -// /** -// * Atrous Convolution module weight initialization test. -// */ -// TEST_CASE("AtrousConvolutionLayerWeightInitializationTest", "[ANNLayerTest]") -// { -// size_t inSize = 2, outSize = 3; -// size_t kernelWidth = 4, kernelHeight = 5; -// AtrousConvolution<> module = AtrousConvolution<>(inSize, outSize, -// kernelWidth, kernelHeight, 6, 7, std::make_tuple(8, 9), -// std::make_tuple(10, 11), 12, 13, 14, 15); -// module.Reset(); -// RandomInitialization().Initialize(module.Weight()); -// module.Bias().ones(); - -// REQUIRE(std::equal(module.Weight().begin(), -// module.Weight().end(), module.Parameters().begin())); - -// REQUIRE(std::equal(module.Bias().begin(), -// module.Bias().end(), module.Parameters().end() - outSize)); - -// REQUIRE(module.Weight().n_rows == kernelWidth); -// REQUIRE(module.Weight().n_cols == kernelHeight); -// REQUIRE(module.Weight().n_slices == inSize * outSize); -// REQUIRE(module.Bias().n_rows == outSize); -// REQUIRE(module.Bias().n_cols == 1); -// REQUIRE(module.Parameters().n_rows -// == (outSize * inSize * kernelWidth * kernelHeight) + outSize); -// } - /** * Convolution module weight initialization test. */ @@ -5604,730 +793,3 @@ TEST_CASE("ConvolutionLayerWeightInitializationTest", "[ANNLayerTest]") REQUIRE(module.Parameters().n_rows == (outSize * inSize * kernelWidth * kernelHeight) + outSize); } - -/** - * Transposed Convolution module weight initialization test. - * -TEST_CASE("TransposedConvolutionWeightInitializationTest", "[ANNLayerTest]") -{ - size_t inSize = 3, outSize = 3; - size_t kernelWidth = 4, kernelHeight = 4; - TransposedConvolution module = TransposedConvolution(inSize, outSize, - kernelWidth, kernelHeight, 1, 1, 1, 1, 5, 5, 6, 6); - module.Reset(); - RandomInitialization().Initialize(module.Weight()); - module.Bias().ones(); - - REQUIRE(std::equal(module.Weight().begin(), - module.Weight().end(), module.Parameters().begin())); - - REQUIRE(std::equal(module.Bias().begin(), - module.Bias().end(), module.Parameters().end() - outSize)); - - REQUIRE(module.Weight().n_rows == kernelWidth); - REQUIRE(module.Weight().n_cols == kernelHeight); - REQUIRE(module.Weight().n_slices == inSize * outSize); - REQUIRE(module.Bias().n_rows == outSize); - REQUIRE(module.Bias().n_cols == 1); - REQUIRE(module.Parameters().n_rows - == (outSize * inSize * kernelWidth * kernelHeight) + outSize); -} -*/ - -/** - * Simple Test for ChannelShuffle layer. - */ -// TEST_CASE("ChannelShuffleLayerTest", "[ANNLayerTest]") -// { -// arma::mat input1, output1, outputExpected1, outputBackward1; -// ChannelShuffle<> module1(2, 2, 6, 2); -// -// input1 << 1 << 13 << arma::endr -// << 2 << 14 << arma::endr -// << 3 << 15 << arma::endr -// << 4 << 16 << arma::endr -// << 5 << 17 << arma::endr -// << 6 << 18 << arma::endr -// << 7 << 19 << arma::endr -// << 8 << 20 << arma::endr -// << 9 << 21 << arma::endr -// << 10 << 22 << arma::endr -// << 11 << 23 << arma::endr -// << 12 << 24 << arma::endr; -// input1.reshape(24, 1); -// // Value calculated using torch.nn.ChannelShuffle(). -// outputExpected1 << 1 << 17 << arma::endr -// << 2 << 18 << arma::endr -// << 3 << 19 << arma::endr -// << 4 << 20 << arma::endr -// << 13 << 9 << arma::endr -// << 14 << 10 << arma::endr -// << 15 << 11 << arma::endr -// << 16 << 12 << arma::endr -// << 5 << 21 << arma::endr -// << 6 << 22 << arma::endr -// << 7 << 23 << arma::endr -// << 8 << 24 << arma::endr; -// outputExpected1.reshape(24, 1); -// // Check the Forward pass of the layer. -// module1.Forward(input1, output1); -// CheckMatrices(output1, outputExpected1); -// -// // Check the Backward pass of the layer. -// module1.Backward(output1, output1, outputBackward1); -// CheckMatrices(input1, outputBackward1); -// -// } - -/** - * Simple Test for PixelShuffle layer. - */ -// TEST_CASE("PixelShuffleLayerTest", "[ANNLayerTest]") -// { -// arma::mat input1, output1, gy1, g1, outputExpected1, gExpected1; -// arma::mat input2, output2, gy2, g2, outputExpected2, gExpected2; -// PixelShuffle<> module1(2, 2, 2, 4); -// PixelShuffle<> module2(2, 2, 2, 4); -// -// // Input is a single image, of size (2,2) and having 4 channels. -// input1 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 -// << 0 << 0 << arma::endr; -// gy1 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8 -// << 12 << 16 << arma::endr; -// -// // Calculated using torch.nn.PixelShuffle(). -// outputExpected1 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0 -// << 0 << 0 << 0 << 0 << arma::endr; -// gExpected1 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12 -// << 6 << 14 << 8 << 16 << arma::endr; -// -// input1 = input1.t(); -// outputExpected1 = outputExpected1.t(); -// gy1 = gy1.t(); -// gExpected1 = gExpected1.t(); -// -// // Check the Forward pass of the layer. -// module1.Forward(input1, output1); -// CheckMatrices(output1, outputExpected1); -// -// // Check the Backward pass of the layer. -// module1.Backward(input1, gy1, g1); -// CheckMatrices(g1, gExpected1); -// -// // Input is a batch of 2 images, each of size (2,2) and having 4 channels. -// input2 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 -// << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0 -// << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr; -// gy2 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8 -// << 12 << 16 << arma::endr << 17 << 21 << 25 << 29 << 18 << 22 << 26 << 30 -// << 19 << 23 << 27 << 31 << 20 << 24 << 28 << 32 << arma::endr; -// -// // Calculated using torch.nn.PixelShuffle(). -// outputExpected2 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0 -// << 0 << 0 << 0 << 0 << arma::endr << 5 << 0 << 7 << 0 << 0 << 0 << 0 << 0 -// << 6 << 0 << 8 << 0 << 0 << 0 << 0 << 0 << arma::endr; -// gExpected2 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12 -// << 6 << 14 << 8 << 16 << arma::endr << 17 << 25 << 19 << 27 << 21 << 29 -// << 23 << 31 << 18 << 26 << 20 << 28 << 22 << 30 << 24 << 32 << arma::endr; -// -// input2 = input2.t(); -// outputExpected2 = outputExpected2.t(); -// gy2 = gy2.t(); -// gExpected2 = gExpected2.t(); -// -// // Check the Forward pass of the layer. -// module2.Forward(input2, output2); -// CheckMatrices(output2, outputExpected2); -// -// // Check the Backward pass of the layer. -// module2.Backward(input2, gy2, g2); -// CheckMatrices(g2, gExpected2); -// } - -/** - * Test that the function that can access the parameters of the - * PixelShuffle layer works. - */ -// TEST_CASE("PixelShuffleLayerParametersTest", "[ANNLayerTest]") -// { -// // Create the layer using the empty constructor. -// PixelShuffle<> layer; -// -// // Set the different input parameters of the layer. -// layer.UpscaleFactor() = 2; -// layer.InputHeight() = 2; -// layer.InputWidth() = 2; -// layer.InputChannels() = 4; -// -// // Make sure we can get the parameters successfully. -// REQUIRE(layer.UpscaleFactor() == 2); -// REQUIRE(layer.InputHeight() == 2); -// REQUIRE(layer.InputWidth() == 2); -// REQUIRE(layer.InputChannels() == 4); -// -// arma::mat input, output; -// // Input is a batch of 2 images, each of size (2,2) and having 4 channels. -// input << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 -// << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0 -// << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr; -// input = input.t(); -// layer.Forward(input, output); -// -// // Check whether output parameters are returned correctly. -// REQUIRE(layer.OutputHeight() == 4); -// REQUIRE(layer.OutputWidth() == 4); -// REQUIRE(layer.OutputChannels() == 1); -// } - -// /** -// * Simple Test for SpatialDropout layer. -// */ -// TEST_CASE("SpatialDropoutLayerTest", "[ANNLayerTest]") -// { -// arma::mat input, output, gy, g, temp; -// arma::mat outputsExpected = arma::zeros(8, 12); -// arma::mat gsExpected = arma::zeros(8, 12); - -// // Set the seed to a random value. -// arma::arma_rng::set_seed_random(); -// SpatialDropout<> module(3, 0.2); - -// // Input is a batch of 2 images, each of size (2,2) and having 4 channels. -// input = { 0.4963, 0.0885, 0.7682, 0.1320, 0.3074, 0.4901, 0.6341, 0.8964, -// 0.4556, 0.3489, 0.6323, 0.4017 }; -// -// gy = { 1, 3, 2, 4, 5, 7, 6, 8, 9, 11, 10, 12 }; -// -// // Following values have been calculated using torch.nn.Dropout2d(p=0.2). -// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; -// outputsExpected.row(0) = temp; -// temp = { 0, 0, 0, 0, 0.3842, 0.6126, 0.7926, 1.1205, 0.5695, 0.4361, 0.7904, -// 0.5021 }; -// outputsExpected.row(1) = temp; -// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0, 0, 0, 0, 0.5695, 0.4361, -// 0.7904, 0.5021 }; -// outputsExpected.row(2) = temp; -// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0.3842, 0.6126, 0.7926, 1.1205, 0, -// 0, 0, 0 }; -// outputsExpected.row(3) = temp; -// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 0.5695, 0.4361, 0.7904, 0.5021 }; -// outputsExpected.row(4) = temp; -// temp = { 0, 0, 0, 0, 0.3842, 0.6126, 0.7926, 1.1205, 0, 0, 0, 0 }; -// outputsExpected.row(5) = temp; -// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0, 0, 0, 0, 0, 0, 0, 0 }; -// outputsExpected.row(6) = temp; -// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0.3842, 0.6126, 0.7926, 1.1205, -// 0.5695, 0.4361, 0.7904, 0.5021 }; -// outputsExpected.row(7) = temp; -// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; -// gsExpected.row(0) = temp; -// temp = { 0, 0, 0, 0, 6.2500, 8.7500, 7.5000, 10.0000, 11.2500, 13.7500, -// 12.5000, 15.0000 }; -// gsExpected.row(1) = temp; -// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 0, 0, 0, 0, 11.2500, 13.7500, -// 12.5000, 15.0000 }; -// gsExpected.row(2) = temp; -// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 6.2500, 8.7500, 7.5000, 10.0000, 0, -// 0, 0, 0 }; -// gsExpected.row(3) = temp; -// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 11.2500, 13.7500, 12.5000, 15.0000 }; -// gsExpected.row(4) = temp; -// temp = { 0, 0, 0, 0, 6.2500, 8.7500, 7.5000, 10.0000, 0, 0, 0, 0 }; -// gsExpected.row(5) = temp; -// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 0, 0, 0, 0, 0, 0, 0, 0 }; -// gsExpected.row(6) = temp; -// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 6.2500, 8.7500, 7.5000, 10.0000, -// 11.2500, 13.7500, 12.5000, 15.0000 }; -// gsExpected.row(7) = temp; - -// input = input.t(); -// gy = gy.t(); -// outputsExpected = outputsExpected.t(); -// gsExpected = gsExpected.t(); - -// // Compute the Forward and Backward passes and store the results. -// module.Forward(input, output); -// module.Backward(input, gy, g); - -// // Check through all possible cases, to find a match and then compare results. -// for (size_t i = 0; i < outputsExpected.n_cols; ++i) -// { -// if (arma::approx_equal(outputsExpected.col(i), output, "absdiff", 1e-1)) -// { -// // Check the correctness of the Forward pass of the layer. -// CheckMatrices(output, outputsExpected.col(i), 1e-1); -// // Check the correctness of the Backward pass of the layer. -// CheckMatrices(g, gsExpected.col(i), 1e-1); -// } -// } - -// // Check if the output is same as input when using deterministic mode. -// module.Deterministic() = true; -// output.clear(); -// module.Forward(input, output); -// CheckMatrices(output, input, 1e-1); -// } - -// /** -// * Test that the function that can access the parameters of the -// * SpatialDropout layer works. -// */ -// TEST_CASE("SpatialDropoutLayerParametersTest", "[ANNLayerTest]") -// { -// // Create the layer using the empty constructor. -// SpatialDropout<> layer; - -// // Set the input parameters. -// layer.Size() = 3; -// layer.Ratio(0.2); - -// // Check whether the input parameters have been set correctly. -// REQUIRE(layer.Size() == 3); -// REQUIRE(layer.Ratio() == 0.2); -// } - -/** - * Simple Positional Encoding layer test. - * -TEST_CASE("SimplePositionalEncodingTest", "[ANNLayerTest]") -{ - const size_t seqLength = 5; - const size_t embedDim = 4; - const size_t batchSize = 2; - - arma::mat input = arma::randu(embedDim * seqLength, batchSize); - arma::mat gy = 0.01 * arma::randu(embedDim * seqLength, batchSize); - arma::mat output, g; - - PositionalEncoding module(embedDim, seqLength); - - // Check Forward function. - module.Forward(input, output); - arma::mat pe = output - input; - CheckMatrices(arma::mean(pe, 1), module.Encoding()); - - // Check Backward function. - module.Backward(input, gy, g); - REQUIRE(std::equal(gy.begin(), gy.end(), g.begin())); -} -*/ - -/** - * Jacobian test for Positional Encoding layer. - * -TEST_CASE("JacobianPositionalEncodingTest", "[ANNLayerTest]") -{ - for (size_t i = 0; i < 5; ++i) - { - const size_t embedDim = 4; - const size_t seqLength = math::RandInt(5, 10); - arma::mat input; - input.set_size(embedDim * seqLength, 1); - - PositionalEncoding module(embedDim, seqLength); - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -} -*/ - -/** - * Simple Multihead Attention test. - * -TEST_CASE("SimpleMultiheadAttentionTest", "[ANNLayerTest]") -{ - size_t tLen = 5; - size_t sLen = tLen; - size_t embedDim = 4; - size_t numHeads = 2; - size_t bsz = 3; - - arma::mat query = 0.1 * arma::randu(embedDim * tLen, bsz); - arma::mat output; - - arma::mat attnMask = arma::zeros(tLen, sLen); - for (size_t i = 0; i < tLen; ++i) - { - for (size_t j = 0; j < sLen; ++j) - { - if (i < j) - attnMask(i, j) = std::numeric_limits::lowest(); - } - } - - arma::mat keyPaddingMask = arma::zeros(1, sLen); - keyPaddingMask(sLen - 1) = std::numeric_limits::lowest(); - - MultiheadAttention module(tLen, sLen, embedDim, numHeads); - module.AttentionMask() = attnMask; - module.KeyPaddingMask() = keyPaddingMask; - module.Reset(); - module.Parameters().randu(); - - // Forward test. - arma::mat input = arma::join_cols(arma::join_cols(query, query), query); - - module.Forward(input, output); - REQUIRE(output.n_rows == embedDim * tLen); - REQUIRE(output.n_cols == bsz); - - // Backward test. - arma::mat gy = 0.01 * arma::randu(embedDim * tLen, bsz); - arma::mat g; - module.Backward(input, gy, g); - REQUIRE(g.n_rows == input.n_rows); - REQUIRE(g.n_cols == input.n_cols); - - // Gradient test. - arma::mat error = 0.05 * arma::randu(embedDim * tLen, bsz); - arma::mat gradient; - module.Gradient(input, error, gradient); - REQUIRE(gradient.n_rows == module.Parameters().n_rows); - REQUIRE(gradient.n_cols == module.Parameters().n_cols); -} -*/ - -/** - * Jacobian MultiheadAttention module test. - * -TEST_CASE("JacobianMultiheadAttentionTest", "[ANNLayerTest]") -{ - // Check when query = key = value. - for (size_t i = 0; i < 5; ++i) - { - const size_t tgtSeqLen = 2; - const size_t embedDim = 4; - const size_t nHeads = 2; - const size_t batchSize = 1; - - arma::mat query = arma::randu(embedDim * tgtSeqLen, batchSize); - arma::mat input = arma::join_cols(arma::join_cols(query, query), query); - - MultiheadAttention module(tgtSeqLen, tgtSeqLen, embedDim, nHeads); - module.Parameters().randu(); - - double error = CustomJacobianTest(module, input); - REQUIRE(error <= 1e-5); - } - - // Check when key = value. - for (size_t i = 0; i < 5; ++i) - { - const size_t tgtSeqLen = 2; - const size_t srcSeqLen = math::RandInt(2, 5); - const size_t embedDim = 4; - const size_t nHeads = 2; - const size_t batchSize = 1; - - arma::mat query = arma::randu(embedDim * tgtSeqLen, batchSize); - arma::mat key = 0.091 * arma::randu(embedDim * srcSeqLen, batchSize); - arma::mat input = arma::join_cols(arma::join_cols(query, key), key); - - MultiheadAttention module(tgtSeqLen, srcSeqLen, embedDim, nHeads); - module.Parameters().randu(); - - double error = CustomJacobianTest(module, input); - REQUIRE(error <= 1e-5); - } - - // Check when query, key and value are not same. - for (size_t i = 0; i < 5; ++i) - { - const size_t tgtSeqLen = 2; - const size_t srcSeqLen = math::RandInt(2, 5); - const size_t embedDim = 4; - const size_t nHeads = 2; - const size_t batchSize = 1; - - arma::mat query = arma::randu(embedDim * tgtSeqLen, batchSize); - arma::mat key = 0.091 * arma::randu(embedDim * srcSeqLen, batchSize); - arma::mat value = 0.045 * arma::randu(embedDim * srcSeqLen, batchSize); - arma::mat input = arma::join_cols(arma::join_cols(query, key), value); - - MultiheadAttention module(tgtSeqLen, srcSeqLen, embedDim, nHeads); - module.Parameters().randu(); - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -} -*/ - -/** - * Numerical gradient test for MultiheadAttention layer. - * -TEST_CASE("GradientMultiheadAttentionTest", "[ANNLayerTest]") -{ - struct GradientFunction - { - GradientFunction() : - tgtSeqLen(2), - srcSeqLen(2), - embedDim(4), - nHeads(2), - vocabSize(5), - batchSize(2) - { - input = arma::randu(embedDim * (tgtSeqLen + 2 * srcSeqLen), batchSize); - target = arma::zeros(vocabSize, batchSize); - for (size_t i = 0; i < target.n_elem; ++i) - { - const size_t label = mlpack::math::RandInt(1, vocabSize); - target(i) = label; - } - - attnMask = arma::zeros(tgtSeqLen, srcSeqLen); - for (size_t i = 0; i < tgtSeqLen; ++i) - { - for (size_t j = 0; j < srcSeqLen; ++j) - { - if (i < j) - attnMask(i, j) = std::numeric_limits::lowest(); - } - } - - keyPaddingMask = arma::zeros(1, srcSeqLen); - keyPaddingMask(srcSeqLen - 1) = std::numeric_limits::lowest(); - - model = new FFN(); - model->ResetData(input, target); - // attnModule = new MultiheadAttention(tgtSeqLen, srcSeqLen, embedDim, - // nHeads); - // attnModule->AttentionMask() = attnMask; - // attnModule->KeyPaddingMask() = keyPaddingMask; - // model->Add(attnModule); - model->Add(tgtSeqLen, srcSeqLen, embedDim, nHeads, - attnMask, keyPaddingMask); - model->Add(embedDim * tgtSeqLen, vocabSize); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, batchSize); - model->Gradient(model->Parameters(), 0, gradient, batchSize); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - // MultiheadAttention* attnModule; - - arma::mat input, target, attnMask, keyPaddingMask; - const size_t tgtSeqLen; - const size_t srcSeqLen; - const size_t embedDim; - const size_t nHeads; - const size_t vocabSize; - const size_t batchSize; - } function; - - REQUIRE(CheckGradient(function) <= 3e-06); -} -*/ - -/** - * Simple tests for instance normalization layer. - * -TEST_CASE("InstanceNormLayerTest", "[ANNLayerTest]") -{ - arma::mat input, result, output, delta, deltaExpected; - arma::mat runningMean, runningVar; - - // Represents 2 images, each having 3 channels, and shape (3,2). - input << 1 << 19 << arma::endr - << 2 << 20 << arma::endr - << 3 << 21 << arma::endr - << 4 << 22 << arma::endr - << 5 << 23 << arma::endr - << 6 << 24 << arma::endr - << 7 << 25 << arma::endr - << 8 << 26 << arma::endr - << 9 << 27 << arma::endr - << 10 << 28 << arma::endr - << 11 << 29 << arma::endr - << 12 << 30 << arma::endr - << 13 << 31 << arma::endr - << 14 << 32 << arma::endr - << 15 << 33 << arma::endr - << 16 << 34 << arma::endr - << 17 << 35 << arma::endr - << 18 << 36 << arma::endr; - - // Output calculated using torch.nn.InstanceNorm2d(). - result << -1.4638 << -1.4638 << arma::endr - << -0.8783 << -0.8783 << arma::endr - << -0.2928 << -0.2928 << arma::endr - << 0.2928 << 0.2928 << arma::endr - << 0.8783 << 0.8783 << arma::endr - << 1.4638 << 1.4638 << arma::endr - << -1.4638 << -1.4638 << arma::endr - << -0.8783 << -0.8783 << arma::endr - << -0.2928 << -0.2928 << arma::endr - << 0.2928 << 0.2928 << arma::endr - << 0.8783 << 0.8783 << arma::endr - << 1.4638 << 1.4638 << arma::endr - << -1.4638 << -1.4638 << arma::endr - << -0.8783 << -0.8783 << arma::endr - << -0.2928 << -0.2928 << arma::endr - << 0.2928 << 0.2928 << arma::endr - << 0.8783 << 0.8783 << arma::endr - << 1.4638 << 1.4638 << arma::endr; - - // Calculated using torch.nn.InstanceNorm2d(). - deltaExpected << 1.8367 << 1.8367 << arma::endr - << 0.3967 << 0.3967 << arma::endr - << 0.0147 << 0.0147 << arma::endr - <<-0.0147 << -0.0147 << arma::endr - <<-0.3967 << -0.3967 << arma::endr - <<-1.8367 << -1.8367 << arma::endr - << 1.8367 << 1.8367 << arma::endr - << 0.3967 << 0.3967 << arma::endr - << 0.0147 << 0.0147 << arma::endr - <<-0.0147 << -0.0147 << arma::endr - <<-0.3967 << -0.3967 << arma::endr - <<-1.8367 << -1.8367 << arma::endr - << 1.8367 << 1.8367 << arma::endr - << 0.3967 << 0.3967 << arma::endr - << 0.0147 << 0.0147 << arma::endr - <<-0.0147 << -0.0147 << arma::endr - <<-0.3967 << -0.3967 << arma::endr - <<-1.8367 << -1.8367 << arma::endr; - - // Check Forward and Backward pass in non-deterministic mode. - InstanceNorm<> module(3, input.n_cols, 1e-5, false, 0.1); - output.zeros(arma::size(input)); - module.Forward(input, output); - CheckMatrices(output, result, 1e-1); - - module.Backward(input, output, delta); - CheckMatrices(delta, deltaExpected, 1e-1); - - runningMean = arma::mat(3, 1); - runningVar = arma::mat(3, 1); - runningMean(0) = 1.2500; - runningMean(1) = 1.8500; - runningMean(2) = 2.4500; - runningVar(0) = 1.2500; - runningVar(1) = 1.2500; - runningVar(2) = 1.2500; - - CheckMatrices(runningMean, module.TrainingMean(), 1e-1); - CheckMatrices(runningVar, module.TrainingVariance(), 1e-1); - - // Check Forward pass in deterministic mode. - InstanceNorm<> module1(3, input.n_cols, 1e-5, false, 0.1); - module1.Deterministic() = true; - output.zeros(arma::size(input)); - module1.Forward(input, output); - - // Calculated using torch.nn.InstanceNorm2d(). - result << 1.0000 << 18.9999 << arma::endr - << 2.0000 << 19.9999 << arma::endr - << 3.0000 << 20.9999 << arma::endr - << 4.0000 << 21.9999 << arma::endr - << 5.0000 << 22.9999 << arma::endr - << 6.0000 << 23.9999 << arma::endr - << 7.0000 << 24.9999 << arma::endr - << 8.0000 << 25.9999 << arma::endr - << 9.0000 << 26.9999 << arma::endr - << 10.0000 << 27.9999 << arma::endr - << 10.9999 << 28.9999 << arma::endr - << 11.9999 << 29.9999 << arma::endr - << 12.9999 << 30.9998 << arma::endr - << 13.9999 << 31.9998 << arma::endr - << 14.9999 << 32.9998 << arma::endr - << 15.9999 << 33.9998 << arma::endr - << 16.9999 << 34.9998 << arma::endr - << 17.9999 << 35.9998 << arma::endr; - - CheckMatrices(output, result, 1e-1); -} -*/ - -/** - * Test that the functions that can access the parameters of the - * Instance Norm layer work. - * -TEST_CASE("InstanceNormLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order : size, eps. - InstanceNorm<> layer(7, 0, 1e-3); - - // Make sure we can get the parameters successfully. - REQUIRE(layer.InputSize() == 7); - REQUIRE(layer.Epsilon() == 1e-3); - - arma::mat runningMean(7, 1, arma::fill::randn); - arma::mat runningVariance(7, 1, arma::fill::randn); - - layer.TrainingVariance() = runningVariance; - layer.TrainingMean() = runningMean; - CheckMatrices(layer.TrainingVariance(), runningVariance); - CheckMatrices(layer.TrainingMean(), runningMean); -} -*/ - -/** - * Instance Norm layer numerical gradient test. - * -TEST_CASE("GradientInstanceNormLayerTest", "[ANNLayerTest]") -{ - // Add function gradient instantiation. - // To make this test robust, check it ten times. - bool pass = false; - for (size_t trial = 0; trial < 10; trial++) - { - struct GradientFunction - { - GradientFunction() - { - input = arma::randn(16, 1024); - arma::mat target; - target.ones(1, 1024); - - model = new FFN(); - model->ResetData(input, target); - model->Add >(); - model->Add >(1, 2, 3, 3, 1, 1, 0, 0, 4, 4); - model->Add > (2, 1024); - model->Add >(2 * 2 * 2, 2); - model->Add >(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1024, false); - model->Gradient(model->Parameters(), 0, gradient, 1024); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - double gradient = CheckGradient(function); - if (gradient < 1e-1) - { - pass = true; - break; - } - } - - REQUIRE(pass); -} -*/ diff --git a/src/mlpack/tests/ann/feedforward_network_2_test.cpp b/src/mlpack/tests/ann/feedforward_network_2_test.cpp deleted file mode 100644 index 262ff83a39..0000000000 --- a/src/mlpack/tests/ann/feedforward_network_2_test.cpp +++ /dev/null @@ -1,143 +0,0 @@ -/** - * @file tests/feedforward_network_2_test.cpp - * @author Marcus Edel - * @author Palash Ahuja - * - * Tests the feed forward network. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ - -#include - -#include -#include -#include -#include -#include - -#include - -#include "../catch.hpp" -#include "../serialization.hpp" -#include "custom_layer.hpp" - -using namespace mlpack; -using namespace mlpack::ann; -using namespace mlpack::kmeans; - -/** - * Train and evaluate a model with the specified structure. - */ -template -void TestNetwork(ModelType& model, - MatType& trainData, - MatType& trainLabels, - MatType& testData, - MatType& testLabels, - const size_t maxEpochs, - const double classificationErrorThreshold) -{ - ens::RMSProp opt(0.01, 32, 0.88, 1e-8, maxEpochs * trainData.n_cols, -1); - model.Train(trainData, trainLabels, opt); - - MatType predictionTemp; - model.Predict(testData, predictionTemp); - MatType prediction = arma::zeros(1, predictionTemp.n_cols); - - for (size_t i = 0; i < predictionTemp.n_cols; ++i) - { - prediction(i) = arma::as_scalar(arma::find( - arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1; - } - - size_t correct = arma::accu(prediction == testLabels); - double classificationError = 1 - double(correct) / testData.n_cols; - REQUIRE(classificationError <= classificationErrorThreshold); -} - -/** - * Train the RBF network on a larger dataset. - */ -TEST_CASE("RBFNetworkTest", "[FeedForwardNetworkTest]") -{ - // Load the dataset. - arma::mat trainData; - if (!data::Load("thyroid_train.csv", trainData)) - FAIL("Cannot open thyroid_train.csv"); - - arma::mat trainLabels = trainData.row(trainData.n_rows - 1); - trainData.shed_row(trainData.n_rows - 1); - - arma::mat trainLabels1 = arma::zeros(3, trainData.n_cols); - for (size_t i = 0; i < trainData.n_cols; i++) - { - trainLabels1.col(i).row((trainLabels(i) - 1)) = 1; - } - - arma::mat testData; - if (!data::Load("thyroid_test.csv", testData)) - FAIL("Cannot open thyroid_test.csv"); - - arma::mat testLabels = testData.row(testData.n_rows - 1); - testData.shed_row(testData.n_rows - 1); - - /* - * Construct a feed forward network with trainData.n_rows input nodes, - * hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The - * network structure looks like: - * - * Input RBF Activation Output - * Layer Layer Layer Layer - * +-----+ +-----+ +-----+ +-----+ - * | | | | | | | | - * | +------>| +------>| +------>| | - * | | | | | | | | - * +-----+ +--+--+ +-----+ +-----+ - */ - arma::mat centroids; - KMeans<> kmeans; - kmeans.Cluster(trainData, 8, centroids); - - FFN model; - model.Add(8, centroids); - model.Add(3); - - // RBFN neural net with MeanSquaredError. - TestNetwork<>(model, trainData, trainLabels1, testData, testLabels, 10, 0.1); - - arma::mat dataset; - dataset.load("mnist_first250_training_4s_and_9s.arm"); - - // Normalize each point since these are images. - for (size_t i = 0; i < dataset.n_cols; ++i) - { - dataset.col(i) /= norm(dataset.col(i), 2); - } - - arma::mat labels = arma::zeros(1, dataset.n_cols); - labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); - - arma::mat labels1 = arma::zeros(2, dataset.n_cols); - - for (size_t i = 0; i < dataset.n_cols; ++i) - { - labels1.col(i).row(labels(i)) = 1; - } - labels += 1; - - arma::mat centroids1; - arma::Row assignments; - KMeans<> kmeans1; - kmeans1.Cluster(dataset, 140, centroids1); - - FFN model1; - model1.Add(140, centroids1, 4.1); - model1.Add(2); - - // RBFN neural net with MeanSquaredError. - TestNetwork<>(model1, dataset, labels1, dataset, labels, 10, 0.1); -} diff --git a/src/mlpack/tests/ann/feedforward_network_test.cpp b/src/mlpack/tests/ann/feedforward_network_test.cpp index a94a94020b..21d3a2ae3b 100644 --- a/src/mlpack/tests/ann/feedforward_network_test.cpp +++ b/src/mlpack/tests/ann/feedforward_network_test.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include @@ -23,6 +24,7 @@ using namespace mlpack; using namespace mlpack::ann; +using namespace mlpack::kmeans; /** * Train and evaluate a model with the specified structure. @@ -155,75 +157,6 @@ TEST_CASE("CheckCopyMovingVanillaNetworkTest", "[FeedForwardNetworkTest]") CheckMoveFunction(model1, trainData, trainLabels, 1); } -/** - * Check whether copying and moving network with Reparametrization is working or not. - * -TEST_CASE("CheckCopyMovingReparametrizationNetworkTest", - "[FeedForwardNetworkTest]") -{ - // Load the dataset. - arma::mat trainData; - data::Load("thyroid_train.csv", trainData, true); - - arma::mat trainLabels = trainData.row(trainData.n_rows - 1); - trainData.shed_row(trainData.n_rows - 1); - - // Construct a feed forward network with trainData.n_rows input nodes, - // followed by a linear layer and then a reparametrization layer. - FFN *model = new FFN; - model->Add(8); - model->Add(false, true, 1); - model->Add(); - - FFN *model1 = new FFN; - model1->Add(8); - model1->Add(false, true, 1); - model1->Add(); - - // Check whether copy constructor is working or not. - CheckCopyFunction(model, trainData, trainLabels); - - // Check whether move constructor is working or not. - CheckMoveFunction(model1, trainData, trainLabels, 1); -} -*/ - -/** - * Check whether copying and moving network with Reparametrization is working or not. - */ -// TEST_CASE("CheckCopyMovingReparametrizationNetworkTest", -// "[FeedForwardNetworkTest]") -// { -// // Load the dataset. -// arma::mat trainData; -// data::Load("thyroid_train.csv", trainData, true); -// -// // Normalize labels to [0, 2]. -// arma::mat trainLabels = trainData.row(trainData.n_rows - 1) - 1; -// trainData.shed_row(trainData.n_rows - 1); -// -// /* -// * Construct a feed forward network with trainData.n_rows input nodes, -// * followed by a linear layer and then a reparametrization layer. -// */ -// -// FFN *model = new FFN; -// model->Add >(trainData.n_rows, 8); -// model->Add >(4, false, true, 1); -// model->Add >(); -// -// FFN *model1 = new FFN; -// model1->Add >(trainData.n_rows, 8); -// model1->Add >(4, false, true, 1); -// model1->Add >(); -// -// // Check whether copy constructor is working or not. -// CheckCopyFunction<>(model, trainData, trainLabels, 1); -// -// // Check whether move constructor is working or not. -// CheckMoveFunction<>(model1, trainData, trainLabels, 1); -// } - /** * Check whether copying and moving network with linear3d is working or not. */ @@ -416,42 +349,6 @@ TEST_CASE("CheckCopyMovingVanillaNetworkTestNoBias", "[FeedForwardNetworkTest]") CheckMoveFunction<>(model1, trainData, trainLabels, 1); } -/** - * Check whether copying and moving network with Reparametrization is working or not. - */ -// TEST_CASE("CheckCopyMovingReparametrizationNetworkTestNoBias", -// "[FeedForwardNetworkTest]") -// { -// // Load the dataset. -// arma::mat trainData; -// data::Load("thyroid_train.csv", trainData, true); -// -// // Normalize labels to [0, 2]. -// arma::mat trainLabels = trainData.row(trainData.n_rows - 1) - 1; -// trainData.shed_row(trainData.n_rows - 1); -// -// /* -// * Construct a feed forward network with trainData.n_rows input nodes, -// * followed by a linear layer and then a reparametrization layer. -// */ -// -// FFN *model = new FFN; -// model->Add >(trainData.n_rows, 8); -// model->Add >(4, false, true, 1); -// model->Add >(); -// -// FFN *model1 = new FFN; -// model1->Add >(trainData.n_rows, 8); -// model1->Add >(4, false, true, 1); -// model1->Add >(); -// -// // Check whether copy constructor is working or not. -// CheckCopyFunction<>(model, trainData, trainLabels, 1); -// -// // Check whether move constructor is working or not. -// CheckMoveFunction<>(model1, trainData, trainLabels, 1); -// } - /** * Train the vanilla network on a larger dataset. */ @@ -677,32 +574,6 @@ TEST_CASE("DropoutNetworkTest", "[FeedForwardNetworkTest]") TestNetwork(model1, dataset, labels, dataset, labels, 10, 0.2); } -/** - * Train the highway network on a larger dataset. - * -TEST_CASE("HighwayNetworkTest", "[FeedForwardNetworkTest]") -{ - arma::mat dataset; - dataset.load("mnist_first250_training_4s_and_9s.arm"); - - // Normalize each point since these are images. - for (size_t i = 0; i < dataset.n_cols; ++i) - dataset.col(i) /= norm(dataset.col(i), 2); - - arma::mat labels = arma::zeros(1, dataset.n_cols); - labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); - - FFN model; - model.Add(10); - Highway* highway = new Highway(); - highway->Add(10); - highway->Add(); - model.Add(highway); // This takes ownership of the memory. - model.Add(2); - model.Add(); - TestNetwork(model, dataset, labels, dataset, labels, 10, 0.2); -}*/ - /** * Train the DropConnect network on a larger dataset. */ @@ -848,95 +719,6 @@ TEST_CASE("FFSerializationTest", "[FeedForwardNetworkTest]") binaryPredictions); } -/** - * Test that serialization works ok for PReLU. - */ -// TEST_CASE("PReLUSerializationTest", "[FeedForwardNetworkTest]") -// { -// // Load the dataset. -// arma::mat trainData; -// if (!data::Load("thyroid_train.csv", trainData)) -// FAIL("Cannot open thyroid_train.csv"); -// -// arma::mat trainLabels = trainData.row(trainData.n_rows - 1); -// trainData.shed_row(trainData.n_rows - 1); -// trainLabels -= 1; // The labels should be between 0 and numClasses - 1. -// -// arma::mat testData; -// if (!data::Load("thyroid_test.csv", testData)) -// FAIL("Cannot load dataset thyroid_test.csv"); -// -// arma::mat testLabels = testData.row(testData.n_rows - 1); -// testData.shed_row(testData.n_rows - 1); -// testLabels -= 1; // The labels should be between 0 and numClasses - 1. -// -// // Vanilla neural net with logistic activation function. -// // Because 92% of the patients are not hyperthyroid the neural -// // network must be significant better than 92%. -// FFN model; -// model.Add >(trainData.n_rows, 8); -// model.Add >(); -// model.Add >(); -// model.Add >(8, 3); -// model.Add >(); -// -// ens::RMSProp opt(0.01, 32, 0.88, 1e-8, trainData.n_cols /* 1 epoch */, -1); -// -// model.Train(trainData, trainLabels, opt); -// -// FFN xmlModel, jsonModel, binaryModel; -// xmlModel.Add>(10, 10); // Layer that will get removed. -// -// // Serialize into other models. -// SerializeObjectAll(model, xmlModel, jsonModel, binaryModel); -// -// arma::mat predictions, xmlPredictions, jsonPredictions, binaryPredictions; -// model.Predict(testData, predictions); -// xmlModel.Predict(testData, xmlPredictions); -// jsonModel.Predict(testData, jsonPredictions); -// jsonModel.Predict(testData, binaryPredictions); -// -// CheckMatrices(predictions, xmlPredictions, jsonPredictions, -// binaryPredictions); -// } - -/** - * Test if the custom layers work. The target is to see if the code compiles - * when the Train and Prediction are called. - */ -// TEST_CASE("CustomLayerTest", "[FeedForwardNetworkTest]") -// { -// // Load the dataset. -// arma::mat trainData; -// if (!data::Load("thyroid_train.csv", trainData)) -// FAIL("Cannot open thyroid_train.csv"); -// -// arma::mat trainLabels = trainData.row(trainData.n_rows - 1); -// trainData.shed_row(trainData.n_rows - 1); -// trainLabels -= 1; // The labels should be between 0 and numClasses - 1. -// -// arma::mat testData; -// if (!data::Load("thyroid_test.csv", testData)) -// FAIL("Cannot load dataset thyroid_test.csv"); -// -// arma::mat testLabels = testData.row(testData.n_rows - 1); -// testData.shed_row(testData.n_rows - 1); -// testLabels -= 1; // The labels should be between 0 and numClasses - 1. -// -// FFN > model; -// model.Add >(trainData.n_rows, 8); -// model.Add >(); -// model.Add >(8, 3); -// model.Add >(); -// -// ens::RMSProp opt(0.01, 32, 0.88, 1e-8, 15, -1); -// model.Train(trainData, trainLabels, opt); -// -// arma::mat predictionTemp; -// model.Predict(testData, predictionTemp); -// arma::mat prediction = arma::zeros(1, predictionTemp.n_cols); -// } - /** * Test the overload of Forward function which allows partial forward pass. */ @@ -1123,3 +905,86 @@ TEST_CASE("FFNCheckInputShapeTest", "[FeedForwardNetworkTest]") REQUIRE_THROWS_AS(model.Train(trainData, trainLabels, opt), std::logic_error); } + +/** + * Train the RBF network on a larger dataset. + */ +TEST_CASE("RBFNetworkTest", "[FeedForwardNetworkTest]") +{ + // Load the dataset. + arma::mat trainData; + if (!data::Load("thyroid_train.csv", trainData)) + FAIL("Cannot open thyroid_train.csv"); + + arma::mat trainLabels = trainData.row(trainData.n_rows - 1); + trainData.shed_row(trainData.n_rows - 1); + + arma::mat trainLabels1 = arma::zeros(3, trainData.n_cols); + for (size_t i = 0; i < trainData.n_cols; i++) + { + trainLabels1.col(i).row((trainLabels(i) - 1)) = 1; + } + + arma::mat testData; + if (!data::Load("thyroid_test.csv", testData)) + FAIL("Cannot open thyroid_test.csv"); + + arma::mat testLabels = testData.row(testData.n_rows - 1); + testData.shed_row(testData.n_rows - 1); + + /* + * Construct a feed forward network with trainData.n_rows input nodes, + * hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The + * network structure looks like: + * + * Input RBF Activation Output + * Layer Layer Layer Layer + * +-----+ +-----+ +-----+ +-----+ + * | | | | | | | | + * | +------>| +------>| +------>| | + * | | | | | | | | + * +-----+ +--+--+ +-----+ +-----+ + */ + arma::mat centroids; + KMeans<> kmeans; + kmeans.Cluster(trainData, 8, centroids); + + FFN model; + model.Add(8, centroids); + model.Add(3); + + // RBFN neural net with MeanSquaredError. + TestNetwork<>(model, trainData, trainLabels1, testData, testLabels, 10, 0.1); + + arma::mat dataset; + dataset.load("mnist_first250_training_4s_and_9s.arm"); + + // Normalize each point since these are images. + for (size_t i = 0; i < dataset.n_cols; ++i) + { + dataset.col(i) /= norm(dataset.col(i), 2); + } + + arma::mat labels = arma::zeros(1, dataset.n_cols); + labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); + + arma::mat labels1 = arma::zeros(2, dataset.n_cols); + + for (size_t i = 0; i < dataset.n_cols; ++i) + { + labels1.col(i).row(labels(i)) = 1; + } + labels += 1; + + arma::mat centroids1; + arma::Row assignments; + KMeans<> kmeans1; + kmeans1.Cluster(dataset, 140, centroids1); + + FFN model1; + model1.Add(140, centroids1, 4.1); + model1.Add(2); + + // RBFN neural net with MeanSquaredError. + TestNetwork<>(model1, dataset, labels1, dataset, labels, 10, 0.1); +} diff --git a/src/mlpack/tests/ann/not_adapted/activation_functions_test.cpp b/src/mlpack/tests/ann/not_adapted/activation_functions_test.cpp new file mode 100644 index 0000000000..7567b3f5da --- /dev/null +++ b/src/mlpack/tests/ann/not_adapted/activation_functions_test.cpp @@ -0,0 +1,817 @@ +/** + * @file tests/activation_functions_test.cpp + * @author Marcus Edel + * @author Dhawal Arora + * + * Tests for the various activation functions. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../catch.hpp" + +using namespace mlpack; +using namespace mlpack::ann; + +/** + * Implementation of the HardTanH activation function test. The function is + * implemented as a HardTanH Layer in hard_tanh.hpp + * + * @param input Input data used for evaluating the HardTanH activation function. + * @param target Target data used to evaluate the HardTanH activation. + * +void CheckHardTanHActivationCorrect(const arma::colvec input, + const arma::colvec target) +{ + HardTanH<> htf; + + // Test the activation function using the entire vector as input. + arma::colvec activations; + htf.Forward(input, activations); + for (size_t i = 0; i < activations.n_elem; ++i) + { + REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Implementation of the HardTanH activation function derivative test. The + * derivative is implemented as HardTanH Layer in hard_tanh.hpp + * + * @param input Input data used for evaluating the HardTanH activation + * function. + * @param target Target data used to evaluate the HardTanH activation. + * +void CheckHardTanHDerivativeCorrect(const arma::colvec input, + const arma::colvec target) +{ + HardTanH<> htf; + + // Test the calculation of the derivatives using the entire vector as input. + arma::colvec derivatives; + + // This error vector will be set to 1 to get the derivatives. + arma::colvec error = arma::ones(input.n_elem); + htf.Backward(input, error, derivatives); + + for (size_t i = 0; i < derivatives.n_elem; ++i) + { + REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Implementation of the ELU activation function test. The function is + * implemented as ELU layer in the file elu.hpp + * + * @param input Input data used for evaluating the ELU activation function. + * @param target Target data used to evaluate the ELU activation. + * +void CheckELUActivationCorrect(const arma::colvec input, + const arma::colvec target) +{ + // Initialize ELU object with alpha = 1.0. + ELU<> lrf(1.0); + + // Test the activation function using the entire vector as input. + arma::colvec activations; + lrf.Forward(input, activations); + for (size_t i = 0; i < activations.n_elem; ++i) + { + REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Implementation of the ELU activation function derivative test. The function + * is implemented as ELU layer in the file elu.hpp + * + * @param input Input data used for evaluating the ELU activation function. + * @param target Target data used to evaluate the ELU activation. + * +void CheckELUDerivativeCorrect(const arma::colvec input, + const arma::colvec target) +{ + // Initialize ELU object with alpha = 1.0. + ELU<> lrf(1.0); + + // Test the calculation of the derivatives using the entire vector as input. + arma::colvec derivatives, activations; + + // This error vector will be set to 1 to get the derivatives. + arma::colvec error = arma::ones(input.n_elem); + lrf.Forward(input, activations); + lrf.Backward(activations, error, derivatives); + for (size_t i = 0; i < derivatives.n_elem; ++i) + { + REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Implementation of the PReLU activation function test. The function + * is implemented as PReLU layer in the file parametric_relu.hpp. + * + * @param input Input data used for evaluating the PReLU activation + * function. + * @param target Target data used to evaluate the PReLU activation. + * +void CheckPReLUActivationCorrect(const arma::colvec input, + const arma::colvec target) +{ + PReLU<> prelu; + + // Test the activation function using the entire vector as input. + arma::colvec activations; + prelu.Forward(input, activations); + for (size_t i = 0; i < activations.n_elem; ++i) + { + REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Implementation of the PReLU activation function derivative test. + * The function is implemented as PReLU layer in the file + * parametric_relu.hpp + * + * @param input Input data used for evaluating the PReLU activation + * function. + * @param target Target data used to evaluate the PReLU activation. + * +void CheckPReLUDerivativeCorrect(const arma::colvec input, + const arma::colvec target) +{ + PReLU<> prelu; + + // Test the calculation of the derivatives using the entire vector as input. + arma::colvec derivatives; + + // This error vector will be set to 1 to get the derivatives. + arma::colvec error = arma::ones(input.n_elem); + prelu.Backward(input, error, derivatives); + for (size_t i = 0; i < derivatives.n_elem; ++i) + { + REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Implementation of the PReLU activation function gradient test. + * The function is implemented as PReLU layer in the file + * parametric_relu.hpp + * + * @param input Input data used for evaluating the PReLU activation + * function. + * @param target Target data used to evaluate the PReLU gradient. + * +void CheckPReLUGradientCorrect(const arma::colvec input, + const arma::colvec target) +{ + PReLU<> prelu; + + // Test the calculation of the derivatives using the entire vector as input. + arma::colvec gradient; + + // This error vector will be set to 1 to get the gradient. + arma::colvec error = arma::ones(input.n_elem); + prelu.Gradient(input, error, gradient); + REQUIRE(gradient.n_rows == 1); + REQUIRE(gradient.n_cols == 1); + REQUIRE(gradient(0) == Approx(target(0)).epsilon(1e-5)); +}*/ + +/** + * Implementation of the Hard Shrink activation function test. The function is + * implemented as Hard Shrink layer in the file hardshrink.hpp + * + * @param input Input data used for evaluating the Hard Shrink activation function. + * @param target Target data used to evaluate the Hard Shrink activation. + * +void CheckHardShrinkActivationCorrect(const arma::colvec input, + const arma::colvec target) +{ + HardShrink<> hardshrink; + + // Test the activation function using the entire vector as input. + arma::colvec activations; + hardshrink.Forward(input, activations); + for (size_t i = 0; i < activations.n_elem; ++i) + { + REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Implementation of the HardShrink activation function derivative test. + * The derivative function is implemented as HardShrink layer in the file + * hardshrink.hpp + * + * @param input Input data used for evaluating the HardShrink activation + * function. + * @param target Target data used to evaluate the HardShrink activation. + * +void CheckHardShrinkDerivativeCorrect(const arma::colvec input, + const arma::colvec target) +{ + HardShrink<> hardshrink; + + // Test the calculation of the derivatives using the entire vector as input. + arma::colvec derivatives; + + // This error vector will be set to 1 to get the derivatives. + arma::colvec error = arma::ones(input.n_elem); + hardshrink.Backward(input, error, derivatives); + for (size_t i = 0; i < derivatives.n_elem; ++i) + { + REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Implementation of the Soft Shrink activation function test. The function is + * implemented as Soft Shrink layer in the file softshrink.hpp. + * + * @param input Input data used for evaluating the Soft Shrink activation + * function. + * @param target Target data used to evaluate the Soft Shrink activation. + * +void CheckSoftShrinkActivationCorrect(const arma::colvec input, + const arma::colvec target) +{ + SoftShrink<> softshrink; + + // Test the activation function using the entire vector as input. + arma::colvec activations; + softshrink.Forward(input, activations); + for (size_t i = 0; i < activations.n_elem; ++i) + { + REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Implementation of the Soft Shrink activation function derivative test. + * The derivative function is implemented as Soft Shrink layer in the file + * softshrink.hpp + * + * @param input Input data used for evaluating the Soft Shrink activation + * function. + * @param target Target data used to evaluate the Soft Shrink activation. + * +void CheckSoftShrinkDerivativeCorrect(const arma::colvec input, + const arma::colvec target) +{ + SoftShrink<> softshrink; + + // Test the calculation of the derivatives using the entire vector as input. + arma::colvec derivatives; + + // This error vector will be set to 1 to get the derivatives. + arma::colvec error = arma::ones(input.n_elem); + softshrink.Backward(input, error, derivatives); + for (size_t i = 0; i < derivatives.n_elem; ++i) + { + REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Simple SELU activation test to check whether the mean and variance remain + * invariant after passing normalized inputs through the function. + * +TEST_CASE("SELUFunctionNormalizedTest", "[ActivationFunctionsTest]") +{ + arma::mat input = arma::randn(1000, 1); + + arma::mat output; + + SELU selu; + + selu.Forward(input, output); + + REQUIRE(arma::as_scalar(arma::abs(arma::mean(input) - + arma::mean(output))) <= 0.1); + + REQUIRE(arma::as_scalar(arma::abs(arma::var(input) - + arma::var(output))) <= 0.1); +}*/ + +/** + * Simple SELU activation test to check whether the mean and variance + * vary significantly after passing unnormalized inputs through the function. + * +TEST_CASE("SELUFunctionUnnormalizedTest", "[ActivationFunctionsTest]") +{ + const arma::colvec input("5.96402758 0.9966824 0.99975321 1 \ + 7.76159416 -0.76159416 0.96402758 8"); + + arma::mat output; + + SELU selu; + + selu.Forward(input, output); + + REQUIRE(arma::as_scalar(arma::abs(arma::mean(input) - + arma::mean(output))) >= 0.1); + + REQUIRE(arma::as_scalar(arma::abs(arma::var(input) - + arma::var(output))) >= 0.1); +}*/ + +/** + * Simple SELU derivative test to check whether the derivatives + * produced by the activation function are correct. + * + * +TEST_CASE("SELUFunctionDerivativeTest", "[ActivationFunctionsTest]") +{ + arma::mat input = arma::ones(1000, 1); + + arma::mat error = arma::ones(input.n_elem, 1); + + arma::mat derivatives, activations; + + SELU selu; + + selu.Forward(input, activations); + selu.Backward(activations, error, derivatives); + + REQUIRE(arma::as_scalar(arma::abs(arma::mean(derivatives) - + selu.Lambda())) <= 10e-4); + + input.fill(-1); + + selu.Forward(input, activations); + selu.Backward(activations, error, derivatives); + + REQUIRE(arma::as_scalar(arma::abs(arma::mean(derivatives) - + selu.Lambda() * selu.Alpha() - arma::mean(activations))) <= 10e-4); +}*/ + +/** + * Implementation of the CELU activation function test. The function is + * implemented as CELU layer in the file celu.hpp. + * + * @param input Input data used for evaluating the CELU activation function. + * @param target Target data used to evaluate the CELU activation. + * +void CheckCELUActivationCorrect(const arma::colvec input, + const arma::colvec target) +{ + // Initialize CELU object with alpha = 1.0. + CELU<> lrf(1.0); + + // Test the activation function using the entire vector as input. + arma::colvec activations; + lrf.Forward(input, activations); + for (size_t i = 0; i < activations.n_elem; ++i) + { + REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Implementation of the CELU activation function derivative test. The function + * is implemented as CELU layer in the file celu.hpp. + * + * @param input Input data used for evaluating the CELU activation function. + * @param target Target data used to evaluate the CELU activation. + * +void CheckCELUDerivativeCorrect(const arma::colvec input, + const arma::colvec target) +{ + // Initialize CELU object with alpha = 1.0. + CELU<> lrf(1.0); + + // Test the calculation of the derivatives using the entire vector as input. + arma::colvec derivatives, activations; + + // This error vector will be set to 1 to get the derivatives. + arma::colvec error = arma::ones(input.n_elem); + lrf.Forward(input, activations); + lrf.Backward(activations, error, derivatives); + for (size_t i = 0; i < derivatives.n_elem; ++i) + { + REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Implementation of the ISRLU activation function test. The function is + * implemented as ISRLU layer in the file isrlu.hpp. + * + * @param input Input data used for evaluating the ISRLU activation function. + * @param target Target data used to evaluate the ISRLU activation. + * +void CheckISRLUActivationCorrect(const arma::colvec input, + const arma::colvec target) +{ + // Initialize ISRLU object with alpha = 1.0. + ISRLU<> lrf(1.0); + + // Test the activation function using the entire vector as input. + arma::colvec activations; + lrf.Forward(input, activations); + for (size_t i = 0; i < activations.n_elem; ++i) + { + REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Implementation of the ISRLU activation function derivative test. The function + * is implemented as ISRLU layer in the file isrlu.hpp. + * + * @param input Input data used for evaluating the ISRLU activation function. + * @param target Target data used to evaluate the ISRLU activation. + * +void CheckISRLUDerivativeCorrect(const arma::colvec input, + const arma::colvec target) +{ + // Initialize ISRLU object with alpha = 1.0. + ISRLU<> lrf(1.0); + + // Test the calculation of the derivatives using the entire vector as input. + arma::colvec derivatives, activations; + + // This error vector will be set to 1 to get the derivatives. + arma::colvec error = arma::ones(input.n_elem); + lrf.Forward(input, activations); + lrf.Backward(activations, error, derivatives); + for (size_t i = 0; i < derivatives.n_elem; ++i) + { + REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Implementation of the Softmin activation function test. The function is + * implemented as Softmin layer in the file softmin.hpp. + * + * @param input Input data used for evaluating the Softmin activation function. + * @param target Target data used to evaluate the Softmin activation. + * +void CheckSoftminActivationCorrect(const arma::colvec input, + const arma::colvec target) +{ + // Initialize Softmin object. + Softmin<> softmin; + + // Test the activation function using the entire vector as input. + arma::colvec activations; + softmin.Forward(input, activations); + for (size_t i = 0; i < activations.n_elem; ++i) + { + REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Implementation of the Softmin activation function derivative test. + * The function is implemented as Softmin layer in the file softmin.hpp. + * + * @param input Input data used for evaluating the Softmin activation function. + * @param target Target data used to evaluate the Softmin activation. + * +void CheckSoftminDerivativeCorrect(const arma::colvec input, + const arma::colvec target) +{ + // Initialize Softmin object. + Softmin<> softmin; + + // Test the calculation of the derivatives using the entire vector as input. + arma::colvec derivatives, activations; + + // This error vector will be set to [[1.0],[0.0],[1.0],[0.0]] + // to get the derivatives. + arma::colvec error = arma::ones(input.n_elem); + error(1) = 0.0; + error(3) = 0.0; + softmin.Forward(input, activations); + softmin.Backward(activations, error, derivatives); + for (size_t i = 0; i < derivatives.n_elem; ++i) + { + REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Implementation of the Flatten T Swish activation function test. The function is + * implemented as Flatten T Swish layer in the file flatten_t_swish.hpp. + * + * @param input Input data used for evaluating the Flatten T Swish activation + * function. + * @param target Target data used to evaluate the Flatten T Swish activation. + * +void CheckFlattenTSwishActivationCorrect(const arma::colvec input, + const arma::colvec target) +{ + FlattenTSwish<> fts(0.4); + arma::colvec activations; + + fts.Forward(input, activations); + for (size_t i = 0; i < activations.n_elem; ++i) + { + REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Implementation of the Softmin activation function derivative test. + * The function is implemented as Softmin layer in the file softmin.hpp. + * + * @param input Input data used for evaluating the Softmin activation function. + * @param target Target data used to evaluate the Softmin activation. + * +void CheckFlattenTSwishDerivateCorrect(const arma::colvec input, + const arma::colvec target) +{ + FlattenTSwish<> fts; + + // Set the error to 1 to get the actual derivative. + arma::colvec error = arma::ones(input.n_elem); + + arma::colvec derivate; + fts.Backward(input, error, derivate); + for (size_t i = 0; i < derivate.n_elem; ++i) + { + REQUIRE(derivate.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Implementation of the ReLU6 activation function derivative test. The function + * is implemented as ReLU6 layer in the file relu6.hpp. + * + * @param input Input data used for evaluating the ReLU6 activation function. + * @param target Target data used to evaluate the ReLU6 activation. + * +void CheckReLU6Correct(const arma::colvec input, + const arma::colvec ActivationTarget, + const arma::colvec DerivativeTarget) +{ + // Initialize ReLU6 object. + ReLU6<> relu6; + + // Test the calculation of the derivatives using the entire vector as input. + arma::colvec derivatives, activations; + + // This error vector will be set to 1 to get the derivatives. + arma::colvec error = arma::ones(input.n_elem); + relu6.Forward(input, activations); + for (size_t i = 0; i < activations.n_elem; ++i) + { + REQUIRE(activations.at(i) == Approx(ActivationTarget.at(i)).epsilon(1e-5)); + } + relu6.Backward(activations, error, derivatives); + for (size_t i = 0; i < derivatives.n_elem; ++i) + { + REQUIRE(derivatives.at(i) == Approx(DerivativeTarget.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Basic test of the ReLU6 function. + * +TEST_CASE("ReLU6FunctionTest", "[ActivationFunctionsTest]") +{ + const arma::colvec activationData("-2.0 3.0 0.0 6.0 24.0"); + + // desiredActivations taken from PyTorch. + const arma::colvec desiredActivations("0.0 3.0 0.0 6.0 6.0"); + + // desiredDerivatives taken from PyTorch. + const arma::colvec desiredDerivatives("0.0 1.0 0.0 0.0 0.0"); + + CheckReLU6Correct(activationData, desiredActivations, desiredDerivatives); +}*/ + +/** + * Basic test of the HardTanH function. + * +TEST_CASE("HardTanHFunctionTest", "[ActivationFunctionsTest]") +{ + const arma::colvec desiredActivations("-1 1 1 -1 \ + 1 -1 1 0"); + + const arma::colvec desiredDerivatives("0 0 0 0 \ + 1 1 0 1"); + + CheckHardTanHActivationCorrect(activationData, desiredActivations); + CheckHardTanHDerivativeCorrect(activationData, desiredDerivatives); +}*/ + +/** + * Basic test of the ELU function. + * +TEST_CASE("ELUFunctionTest", "[ActivationFunctionsTest]") +{ + const arma::colvec desiredActivations("-0.86466471 3.2 4.5 -1.0 \ + 1 -0.63212055 2 0"); + + const arma::colvec desiredDerivatives("0.13533529 1 1 0 \ + 1 0.36787945 1 1"); + + CheckELUActivationCorrect(activationData, desiredActivations); + CheckELUDerivativeCorrect(activationData, desiredDerivatives); +}*/ + +/** + * Basic test of the PReLU function. + * +TEST_CASE("PReLUFunctionTest", "[ActivationFunctionsTest]") +{ + const arma::colvec desiredActivations("-0.06 3.2 4.5 -3.006 \ + 1 -0.03 2 0"); + + const arma::colvec desiredDerivatives("0.03 1 1 0.03 \ + 1 0.03 1 1"); + const arma::colvec desiredGradient("-103.2"); + + CheckPReLUActivationCorrect(activationData, desiredActivations); + CheckPReLUDerivativeCorrect(desiredActivations, desiredDerivatives); + CheckPReLUGradientCorrect(activationData, desiredGradient); +}*/ + +/** + * Basic test of the CReLU function. + * +TEST_CASE("CReLUFunctionTest", "[ActivationFunctionsTest]") +{ + const arma::colvec desiredActivations("0 3.2 4.5 0 \ + 1 0 2 0 2 0 0 \ + 100.2 0 1 0 0"); + + const arma::colvec desiredDerivatives("0 0 0 0 \ + 0 0 0 0"); + CReLU<> crelu; + // Test the activation function using the entire vector as input. + arma::colvec activations; + crelu.Forward(activationData, activations); + arma::colvec derivatives; + // This error vector will be set to 1 to get the derivatives. + arma::colvec error = arma::ones(desiredActivations.n_elem); + crelu.Backward(desiredActivations, error, derivatives); + for (size_t i = 0; i < activations.n_elem; ++i) + { + REQUIRE(activations.at(i) == + Approx(desiredActivations.at(i)).epsilon(1e-5)); + } + for (size_t i = 0; i < derivatives.n_elem; ++i) + { + REQUIRE(derivatives.at(i) == + Approx(desiredDerivatives.at(i)).epsilon(1e-5)); + } +}*/ + +/** + * Basic test of the swish function. + * +TEST_CASE("SwishFunctionTest", "[ActivationFunctionsTest]") +{ + // Hand-calculated values using Python interpreter. + const arma::colvec desiredActivations("-0.238405 3.07466 4.45055 \ + -3.05183208657e-42 0.731058 -0.26894 \ + 1.76159 0"); + + const arma::colvec desiredDerivatives("0.3819171 1.0856295 1.039218 \ + 0.5 0.83540367 0.3671335 1.073787\ + 0.5"); + + CheckActivationCorrect(activationData, desiredActivations); + CheckDerivativeCorrect(desiredActivations, + desiredDerivatives); +}*/ + +/** + * Basic test of the Hard Shrink function. + * +TEST_CASE("HardShrinkFunctionTest", "[ActivationFunctionsTest]") +{ + const arma::colvec desiredActivations("-2 3.2 4.5 -100.2 1 -1 2 0"); + + const arma::colvec desiredDerivatives("1 1 1 1 1 1 1 0"); + + CheckHardShrinkActivationCorrect(activationData, + desiredActivations); + CheckHardShrinkDerivativeCorrect(desiredActivations, + desiredDerivatives); +}*/ + +/** + * Basic test of the Soft Shrink function. + * +TEST_CASE("SoftShrinkFunctionTest", "[ActivationFunctionsTest]") +{ + const arma::colvec desiredActivations("-1.5 2.7 4 -99.7 0.5 -0.5 1.5 0"); + + const arma::colvec desiredDerivatives("1 1 1 1 1 1 1 0"); + + CheckSoftShrinkActivationCorrect(activationData, + desiredActivations); + CheckSoftShrinkDerivativeCorrect(desiredActivations, + desiredDerivatives); +}*/ + +/** + * Basic test of the CELU activation function. + * +TEST_CASE("CELUFunctionTest", "[ActivationFunctionsTest]") +{ + const arma::colvec desiredActivations("-0.86466472 3.2 4.5 \ + -1 1 -0.63212056 2 0"); + + const arma::colvec desiredDerivatives("0.42119275 1 1 \ + 0.36787944 1 \ + 0.5314636 1 1"); + + CheckCELUActivationCorrect(activationData, desiredActivations); + CheckCELUDerivativeCorrect(desiredActivations, desiredDerivatives); +}*/ + +/** + * Basic test of the ISRLU activation function. + * +TEST_CASE("ISRLUFunctionTest", "[ActivationFunctionsTest]") +{ + const arma::colvec desiredActivations("-0.89442719 3.2 4.5 \ + -0.99995020 1 -0.70710678 2 0"); + + const arma::colvec desiredDerivatives("0.41408666 1 1 \ + 0.35357980 1 \ + 0.54433105 1 1"); + + CheckISRLUActivationCorrect(activationData, desiredActivations); + CheckISRLUDerivativeCorrect(activationData, desiredDerivatives); +}*/ + +/** + * Basic test of the Softmin function. + * +TEST_CASE("SoftminFunctionTest", "[ActivationFunctionsTest]") +{ + const arma::colvec activationData("4.2 2.4 7.0 6.4"); + + // Hand-calculated Values. + const arma::colvec desiredActivations("0.1384799751 0.8377550303 \ + 0.008420976 0.0153440186"); + + const arma::colvec desiredDerivatives("0.1181371351 -0.12306701070 \ + 0.0071839266 -0.0022540509"); + + CheckSoftminActivationCorrect(activationData, + desiredActivations); + CheckSoftminDerivativeCorrect(activationData, + desiredDerivatives); +}*/ + +/** + * Basic test of Flatten T Swish function. + * +TEST_CASE("FlattenTSwishFunctionTest", "[ActivationFunctionsTest]") +{ + // Random Value. + arma::colvec input("-4.0 -1.0 2 3 4 5 6"); + + // Hand Calculated and using PyTorch. + arma::colvec desiredActivation( + "0.4000000059604645 0.4000000059604645 2.1615941524505615 \ + 3.2577223777770996 4.328054904937744 5.3665361404418945 \ + 6.385164737701416"); + + // Hand Calculated and using PyTorch. + arma::colvec desiredDerivation("0.694792 0.694792 1.096893 1.079178 1.042602 \ + 1.020182 1.009048"); + + CheckFlattenTSwishActivationCorrect(input, desiredActivation); + CheckFlattenTSwishDerivateCorrect(desiredActivation, desiredDerivation); +}*/ diff --git a/src/mlpack/tests/ann/not_adapted/ann_layer_test.cpp b/src/mlpack/tests/ann/not_adapted/ann_layer_test.cpp new file mode 100644 index 0000000000..e4da1e1902 --- /dev/null +++ b/src/mlpack/tests/ann/not_adapted/ann_layer_test.cpp @@ -0,0 +1,5569 @@ +/** + * @file tests/ann_layer_test.cpp + * @author Marcus Edel + * @author Praveen Ch + * + * Tests the ann layer modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../test_catch_tools.hpp" +#include "../catch.hpp" +#include "../serialization.hpp" +#include "ann_test_tools.hpp" + +using namespace mlpack; +using namespace mlpack::ann; + +// // network1 should be allocated with `new`, and trained on some data. +// template +// void CheckRNNCopyFunction(ModelType* network1, +// MatType& trainData, +// MatType& trainLabels, +// const size_t maxEpochs) +// { +// arma::cube predictions1; +// arma::cube predictions2; +// ens::StandardSGD opt(0.1, 1, maxEpochs * trainData.n_slices, -100, false); + +// network1->Train(trainData, trainLabels, opt); +// network1->Predict(trainData, predictions1); + +// RNN<> network2 = *network1; +// delete network1; + +// // Deallocating all of network1's memory, so that network2 does not use any +// // of that memory. +// network2.Predict(trainData, predictions2); +// CheckMatrices(predictions1, predictions2); +// } + +// // network1 should be allocated with `new`, and trained on some data. +// template +// void CheckRNNMoveFunction(ModelType* network1, +// MatType& trainData, +// MatType& trainLabels, +// const size_t maxEpochs) +// { +// arma::cube predictions1; +// arma::cube predictions2; +// ens::StandardSGD opt(0.1, 1, maxEpochs * trainData.n_slices, -100, false); + +// network1->Train(trainData, trainLabels, opt); +// network1->Predict(trainData, predictions1); + +// RNN<> network2(std::move(*network1)); +// delete network1; + +// // Deallocating all of network1's memory, so that network2 does not use any +// // of that memory. +// network2.Predict(trainData, predictions2); +// CheckMatrices(predictions1, predictions2); +// } + +/** + * Simple add module test. + * +TEST_CASE("SimpleAddLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + Add module(10); + module.Parameters().randu(); + + // Test the Forward function. + input = arma::zeros(10, 1); + module.Forward(input, output); + REQUIRE(arma::accu(module.Parameters()) == arma::accu(output)); + + // Test the Backward function. + module.Backward(input, output, delta); + REQUIRE(arma::accu(output) == arma::accu(delta)); + + // Test the forward function. + input = arma::ones(10, 1); + module.Forward(input, output); + REQUIRE(10 + arma::accu(module.Parameters()) == + Approx(arma::accu(output)).epsilon(1e-5)); + + // Test the backward function. + module.Backward(input, output, delta); + REQUIRE(arma::accu(output) == Approx(arma::accu(delta)).epsilon(1e-5)); +} +*/ + +/** + * Jacobian add module test. + * +TEST_CASE("JacobianAddLayerTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t elements = math::RandInt(2, 1000); + arma::mat input; + input.set_size(elements, 1); + + Add module(elements); + module.Parameters().randu(); + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} +*/ + +/** + * Add layer numerical gradient test. + * +TEST_CASE("GradientAddLayerTest", "[ANNLayerTest]") +{ + // Add function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(10, 1)), + target(arma::mat("0")) + { + model = new FFN(); + model->ResetData(input, target); + model->Add(); + model->Add(10, 10); + model->Add(10); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + REQUIRE(CheckGradient(function) <= 1e-4); +}*/ + +/** + * Test that the function that can access the outSize parameter of + * the Add layer works. + * +TEST_CASE("AddLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter : outSize. + Add layer(7); + + // Make sure we can get the parameter successfully. + REQUIRE(layer.OutputSize() == 7); +}*/ + +/** + * Simple constant module test. + * +TEST_CASE("SimpleConstantLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + Constant module(10, 3.0); + + // Test the Forward function. + input = arma::zeros(10, 1); + module.Forward(input, output); + REQUIRE(arma::accu(output) == 30.0); + + // Test the Backward function. + module.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 0); + + // Test the forward function. + input = arma::ones(10, 1); + module.Forward(input, output); + REQUIRE(arma::accu(output) == 30.0); + + // Test the backward function. + module.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 0); +}*/ + +/** + * Jacobian constant module test. + * +TEST_CASE("JacobianConstantLayerTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t elements = math::RandInt(2, 1000); + arma::mat input; + input.set_size(elements, 1); + + Constant module(elements, 1.0); + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +}*/ + +/** + * Test that the function that can access the outSize parameter of the + * Constant layer works. + * +TEST_CASE("ConstantLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter : outSize. + Constant layer(7); + + // Make sure we can get the parameter successfully. + REQUIRE(layer.OutSize() == 7); +}*/ + +// /** +// * Simple linear module test. +// */ +// TEST_CASE("SimpleLinearLayerTest", "[ANNLayerTest]") +// { +// arma::mat output, input, delta; +// Linear<> module(10, 10); +// module.Parameters().randu(); +// module.Reset(); + +// // Test the Forward function. +// input = arma::zeros(10, 1); +// module.Forward(input, output); +// REQUIRE(arma::accu(module.Parameters().submat(100, +// 0, module.Parameters().n_elem - 1, 0)) == +// Approx(arma::accu(output)).epsilon(1e-5)); + +// // Test the Backward function. +// module.Backward(input, input, delta); +// REQUIRE(arma::accu(delta) == 0); +// } + +// /** +// * Jacobian linear module test. +// */ +// TEST_CASE("JacobianLinearLayerTest", "[ANNLayerTest]") +// { +// for (size_t i = 0; i < 5; ++i) +// { +// const size_t inputElements = math::RandInt(2, 1000); +// const size_t outputElements = math::RandInt(2, 1000); + +// arma::mat input; +// input.set_size(inputElements, 1); + +// Linear<> module(inputElements, outputElements); +// module.Parameters().randu(); + +// double error = JacobianTest(module, input); +// REQUIRE(error <= 1e-5); +// } +// } + +// /** +// * Linear layer numerical gradient test. +// */ +// TEST_CASE("GradientLinearLayerTest", "[ANNLayerTest]") +// { +// // Linear function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(10, 1)), +// target(arma::mat("1")) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(10, 10); +// model->Add >(10, 2); +// model->Add >(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// FFN* model; +// arma::mat input, target; +// } function; + +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +// /** +// * Simple noisy linear module test. +// */ +// TEST_CASE("SimpleNoisyLinearLayerTest", "[ANNLayerTest]") +// { +// arma::mat output, input, delta; +// NoisyLinear<> module(10, 10); +// module.Parameters().randu(); +// module.Reset(); + +// // Test the Backward function. +// module.Backward(input, input, delta); +// REQUIRE(arma::accu(delta) == 0); +// } + +// /** +// * Jacobian noisy linear module test. +// */ +// TEST_CASE("JacobianNoisyLinearLayerTest", "[ANNLayerTest]") +// { +// const size_t inputElements = math::RandInt(2, 1000); +// const size_t outputElements = math::RandInt(2, 1000); + +// arma::mat input; +// input.set_size(inputElements, 1); + +// NoisyLinear<> module(inputElements, outputElements); +// module.Parameters().randu(); + +// double error = JacobianTest(module, input); +// REQUIRE(error <= 1e-5); +// } + +// /** +// * Noisy Linear layer numerical gradient test. +// */ +// TEST_CASE("GradientNoisyLinearLayerTest", "[ANNLayerTest]") +// { +// // Noisy linear function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(10, 1)), +// target(arma::mat("1")) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(10, 10); +// model->Add >(10, 2); +// model->Add >(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// FFN* model; +// arma::mat input, target; +// } function; + +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +// /** +// * Jacobian negative log likelihood module test. +// */ +// TEST_CASE("JacobianNegativeLogLikelihoodLayerTest", "[ANNLayerTest]") +// { +// for (size_t i = 0; i < 5; ++i) +// { +// NegativeLogLikelihood module; +// const size_t inputElements = math::RandInt(5, 100); +// arma::mat input; +// RandomInitialization init(0, 1); +// init.Initialize(input, inputElements, 1); + +// arma::mat target(1, 1); +// target(0) = math::RandInt(0, inputElements - 2); + +// double error = JacobianPerformanceTest(module, input, target); +// REQUIRE(error <= 1e-5); +// } +// } + +/** + * Jacobian LeakyReLU module test. + * +TEST_CASE("JacobianLeakyReLULayerTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t inputElements = math::RandInt(2, 1000); + + arma::mat input; + input.set_size(inputElements, 1); + + LeakyReLU module; + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} +*/ + +/** + * Jacobian FlexibleReLU module test. + * +TEST_CASE("JacobianFlexibleReLULayerTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t inputElements = math::RandInt(2, 1000); + + arma::mat input; + input.set_size(inputElements, 1); + + FlexibleReLU module; + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} +*/ + +/** + * Flexible ReLU layer numerical gradient test. + * +TEST_CASE("GradientFlexibleReLULayerTest", "[ANNLayerTest]") +{ + // Add function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(2, 1)), + target(arma::mat("0")) + { + model = new FFN( + NegativeLogLikelihood(), RandomInitialization(0.1, 0.5)); + + model->ResetData(input, target); + model->Add(2, 2); + model->Add(2, 5); + model->Add(0.05); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + REQUIRE(CheckGradient(function) <= 1e-4); +} +*/ + +/** + * Jacobian MultiplyConstant module test. + * +TEST_CASE("JacobianMultiplyConstantLayerTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t inputElements = math::RandInt(2, 1000); + + arma::mat input; + input.set_size(inputElements, 1); + + MultiplyConstant module(3.0); + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} +*/ + +/** + * Check whether copying and moving network with MultiplyConstant is working or + * not. + */ +// TEST_CASE("CheckCopyMoveMultiplyConstantTest", "[ANNLayerTest]") +// { +// arma::mat input(2, 1000); +// input.randu(); +// +// arma::mat output1; +// arma::mat output2; +// arma::mat output3; +// arma::mat output4; +// +// MultiplyConstant<> *module1 = new MultiplyConstant<>(3.0); +// module1->Forward(input, output1); +// +// MultiplyConstant<> module2 = *module1; +// delete module1; +// +// module2.Forward(input, output2); +// CheckMatrices(output1, output2); +// +// MultiplyConstant<> *module3 = new MultiplyConstant<>(3.0); +// module3->Forward(input, output3); +// +// MultiplyConstant<> module4(std::move(*module3)); +// delete module3; +// +// module4.Forward(input, output4); +// CheckMatrices(output3, output4); +// } + +/** + * Jacobian HardTanH module test. + * +TEST_CASE("JacobianHardTanHLayerTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t inputElements = math::RandInt(2, 1000); + + arma::mat input; + input.set_size(inputElements, 1); + + HardTanH module; + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} +*/ + +/** + * Simple select module test. + * +TEST_CASE("SimpleSelectLayerTest", "[ANNLayerTest]") +{ + // TODO: this needs to be adapted + arma::mat outputA, outputB, input, delta; + + input = arma::ones(10, 5); + for (size_t i = 0; i < input.n_cols; ++i) + { + input.col(i) *= i; + } + + // Test the Forward function. + Select moduleA(3); + moduleA.Forward(input, outputA); + REQUIRE(30 == arma::accu(outputA)); + + // Test the Forward function. + Select moduleB(3, 5); + moduleB.Forward(input, outputB); + REQUIRE(15 == arma::accu(outputB)); + + // Test the Backward function. + moduleA.Backward(input, outputA, delta); + REQUIRE(30 == arma::accu(delta)); + + // Test the Backward function. + moduleB.Backward(input, outputA, delta); + REQUIRE(15 == arma::accu(delta)); +} +*/ + +/** + * Test that the functions that can access the parameters of the + * Select layer work. + * +TEST_CASE("SelectLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : index, elements. + Select layer(3, 5); + + // Make sure we can get the parameters successfully. + REQUIRE(layer.Index() == 3); + REQUIRE(layer.NumElements() == 5); +} +*/ + +/** + * Simple join module test. + * +TEST_CASE("SimpleJoinLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + input = arma::ones(10, 5); + + // Test the Forward function. + Join module; + module.Forward(input, output); + REQUIRE(50 == arma::accu(output)); + + bool b = output.n_rows == 1 || output.n_cols == 1; + REQUIRE(b == true); + + // Test the Backward function. + module.Backward(input, output, delta); + REQUIRE(50 == arma::accu(delta)); + + b = delta.n_rows == input.n_rows && input.n_cols; + REQUIRE(b == true); +} +*/ + +// /** +// * Simple add merge module test. +// */ +// TEST_CASE("SimpleAddMergeLayerTest", "[ANNLayerTest]") +// { +// arma::mat output, input, delta; +// input = arma::ones(10, 1); + +// for (size_t i = 0; i < 5; ++i) +// { +// AddMerge<> module(false, false); +// const size_t numMergeModules = math::RandInt(2, 10); +// for (size_t m = 0; m < numMergeModules; ++m) +// { +// IdentityLayer<> identityLayer; +// identityLayer.Forward(input, identityLayer.OutputParameter()); + +// module.Add >(identityLayer); +// } + +// // Test the Forward function. +// module.Forward(input, output); +// REQUIRE(10 * numMergeModules == arma::accu(output)); + +// // Test the Backward function. +// module.Backward(input, output, delta); +// REQUIRE(arma::accu(output) == arma::accu(delta)); +// } +// } + +// /** +// * Test the LSTM layer with a user defined rho parameter and without. +// */ +// TEST_CASE("LSTMRrhoTest", "[ANNLayerTest]") +// { +// const size_t rho = 5; +// arma::cube input = arma::randu(1, 1, 5); +// arma::cube target = arma::ones(1, 1, 5); +// RandomInitialization init(0.5, 0.5); + +// // Create model with user defined rho parameter. +// RNN modelA( +// rho, false, NegativeLogLikelihood(), init); +// modelA.Add >(); +// modelA.Add >(1, 10); + +// // Use LSTM layer with rho. +// modelA.Add >(10, 3, rho); +// modelA.Add >(); + +// // Create model without user defined rho parameter. +// RNN modelB( +// rho, false, NegativeLogLikelihood(), init); +// modelB.Add >(); +// modelB.Add >(1, 10); + +// // Use LSTM layer with rho = MAXSIZE. +// modelB.Add >(10, 3); +// modelB.Add >(); + +// ens::StandardSGD opt(0.1, 1, 5, -100, false); +// modelA.Train(input, target, opt); +// modelB.Train(input, target, opt); + +// CheckMatrices(modelB.Parameters(), modelA.Parameters()); +// } + +// /** +// * LSTM layer numerical gradient test. +// */ +// TEST_CASE("GradientLSTMLayerTest", "[ANNLayerTest]") +// { +// // LSTM function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(1, 1, 5)), +// target(arma::ones(1, 1, 5)) +// { +// const size_t rho = 5; + +// model = new RNN(rho); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(1, 10); +// model->Add >(10, 3, rho); +// model->Add >(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// RNN* model; +// arma::cube input, target; +// } function; + +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +// /** +// * Test that the functions that can modify and access the parameters of the +// * LSTM layer work. +// */ +// TEST_CASE("LSTMLayerParametersTest", "[ANNLayerTest]") +// { +// // Parameter order : inSize, outSize, rho. +// LSTM<> layer1(1, 2, 3); +// LSTM<> layer2(1, 2, 4); + +// // Make sure we can get the parameters successfully. +// REQUIRE(layer1.InSize() == 1); +// REQUIRE(layer1.OutSize() == 2); +// REQUIRE(layer1.Rho() == 3); + +// // Now modify the parameters to match the second layer. +// layer1.Rho() = 4; + +// // Now ensure all the results are the same. +// REQUIRE(layer1.InSize() == layer2.InSize()); +// REQUIRE(layer1.OutSize() == layer2.OutSize()); +// REQUIRE(layer1.Rho() == layer2.Rho()); +// } + +// /** +// * Test the FastLSTM layer with a user defined rho parameter and without. +// */ +// TEST_CASE("FastLSTMRrhoTest", "[ANNLayerTest]") +// { +// const size_t rho = 5; +// arma::cube input = arma::randu(1, 1, 5); +// arma::cube target = arma::ones(1, 1, 5); +// RandomInitialization init(0.5, 0.5); + +// // Create model with user defined rho parameter. +// RNN modelA( +// rho, false, NegativeLogLikelihood(), init); +// modelA.Add >(); +// modelA.Add >(1, 10); + +// // Use FastLSTM layer with rho. +// modelA.Add >(10, 3, rho); +// modelA.Add >(); + +// // Create model without user defined rho parameter. +// RNN modelB( +// rho, false, NegativeLogLikelihood(), init); +// modelB.Add >(); +// modelB.Add >(1, 10); + +// // Use FastLSTM layer with rho = MAXSIZE. +// modelB.Add >(10, 3); +// modelB.Add >(); + +// ens::StandardSGD opt(0.1, 1, 5, -100, false); +// modelA.Train(input, target, opt); +// modelB.Train(input, target, opt); + +// CheckMatrices(modelB.Parameters(), modelA.Parameters()); +// } + +// /** +// * FastLSTM layer numerical gradient test. +// */ +// TEST_CASE("GradientFastLSTMLayerTest", "[ANNLayerTest]") +// { +// // Fast LSTM function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(1, 1, 5)), +// target(arma::ones(1, 1, 5)) +// { +// const size_t rho = 5; + +// model = new RNN(rho); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(1, 10); +// model->Add >(10, 3, rho); +// model->Add >(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// RNN* model; +// arma::cube input, target; +// } function; + +// // The threshold should be << 0.1 but since the Fast LSTM layer uses an +// // approximation of the sigmoid function the estimated gradient is not +// // correct. +// REQUIRE(CheckGradient(function) <= 0.2); +// } + +// /** +// * Test that the functions that can modify and access the parameters of the +// * Fast LSTM layer work. +// */ +// TEST_CASE("FastLSTMLayerParametersTest", "[ANNLayerTest]") +// { +// // Parameter order : inSize, outSize, rho. +// FastLSTM<> layer1(1, 2, 3); +// FastLSTM<> layer2(1, 2, 4); + +// // Make sure we can get the parameters successfully. +// REQUIRE(layer1.InSize() == 1); +// REQUIRE(layer1.OutSize() == 2); +// REQUIRE(layer1.Rho() == 3); + +// // Now modify the parameters to match the second layer. +// layer1.Rho() = 4; + +// // Now ensure all the results are the same. +// REQUIRE(layer1.InSize() == layer2.InSize()); +// REQUIRE(layer1.OutSize() == layer2.OutSize()); +// REQUIRE(layer1.Rho() == layer2.Rho()); +// } + +// /** +// * Check whether copying and moving network with FastLSTM is working or not. +// */ +// TEST_CASE("CheckCopyMoveFastLSTMTest", "[ANNLayerTest]") +// { +// arma::cube input = arma::randu(1, 1, 5); +// arma::cube target = arma::ones(1, 1, 5); +// const size_t rho = 5; + +// RNN *model1 = +// new RNN(rho); +// model1->ResetData(input, target); +// model1->Add >(); +// model1->Add >(1, 10); +// model1->Add >(10, 3, rho); +// model1->Add >(); + +// RNN *model2 = +// new RNN(rho); +// model2->ResetData(input, target); +// model2->Add >(); +// model2->Add >(1, 10); +// model2->Add >(10, 3, rho); +// model2->Add >(); + +// // Check whether copy constructor is working or not. +// CheckRNNCopyFunction<>(model1, input, target, 1); + +// // Check whether move constructor is working or not. +// CheckRNNMoveFunction<>(model2, input, target, 1); +// } + +// /** +// * Check whether copying and moving network with LSTM is working or not. +// */ +// TEST_CASE("CheckCopyMoveLSTMTest", "[ANNLayerTest]") +// { +// arma::cube input = arma::randu(1, 1, 5); +// arma::cube target = arma::ones(1, 1, 5); +// const size_t rho = 5; + +// RNN *model1 = +// new RNN(rho); +// model1->ResetData(input, target); +// model1->Add >(); +// model1->Add >(1, 10); +// model1->Add >(10, 3, rho); +// model1->Add >(); + +// RNN *model2 = +// new RNN(rho); +// model2->ResetData(input, target); +// model2->Add >(); +// model2->Add >(1, 10); +// model2->Add >(10, 3, rho); +// model2->Add >(); + +// // Check whether copy constructor is working or not. +// CheckRNNCopyFunction<>(model1, input, target, 1); + +// // Check whether move constructor is working or not. +// CheckRNNMoveFunction<>(model2, input, target, 1); +// } + +// /** +// * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell +// * state. Besides output, the overloaded function provides read access to cell +// * state of the LSTM layer. +// */ +// TEST_CASE("ReadCellStateParamLSTMLayerTest", "[ANNLayerTest]") +// { +// const size_t rho = 5, inputSize = 3, outputSize = 2; + +// // Provide input of all ones. +// arma::cube input = arma::ones(inputSize, outputSize, rho); + +// arma::mat inputGate, forgetGate, outputGate, hidden; +// arma::mat outLstm, cellLstm; + +// // LSTM layer. +// LSTM<> lstm(inputSize, outputSize, rho); +// lstm.Reset(); +// lstm.ResetCell(rho); + +// // Initialize the weights to all ones. +// lstm.Parameters().ones(); + +// arma::mat inputWeight = arma::ones(outputSize, inputSize); +// arma::mat outputWeight = arma::ones(outputSize, outputSize); +// arma::mat bias = arma::ones(outputSize, input.n_cols); +// arma::mat cellCalc = arma::zeros(outputSize, input.n_cols); +// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); + +// for (size_t seqNum = 0; seqNum < rho; ++seqNum) +// { +// // Wrap a matrix around our data to avoid a copy. +// arma::mat stepData(input.slice(seqNum).memptr(), +// input.n_rows, input.n_cols, false, true); + +// // Apply Forward() on LSTM layer. +// lstm.Forward(stepData, // Input. +// outLstm, // Output. +// cellLstm, // Cell state. +// false); // Don't write into the cell state. + +// // Compute the value of cell state and output. +// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); + +// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); + +// // z = tanh(W.dot(x) + W.dot(h) + b). +// hidden = arma::tanh(inputWeight * stepData + +// outputWeight * outCalc + bias); + +// // c = f * c + i * z. +// cellCalc = forgetGate % cellCalc + inputGate % hidden; + +// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); + +// // h = o * tanh(c). +// outCalc = outputGate % arma::tanh(cellCalc); + +// CheckMatrices(outLstm, outCalc, 1e-12); +// CheckMatrices(cellLstm, cellCalc, 1e-12); +// } +// } + +// /** +// * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell +// * state. Besides output, the overloaded function provides write access to cell +// * state of the LSTM layer. +// */ +// TEST_CASE("WriteCellStateParamLSTMLayerTest", "[ANNLayerTest]") +// { +// const size_t rho = 5, inputSize = 3, outputSize = 2; + +// // Provide input of all ones. +// arma::cube input = arma::ones(inputSize, outputSize, rho); + +// arma::mat inputGate, forgetGate, outputGate, hidden; +// arma::mat outLstm, cellLstm; +// arma::mat cellCalc; + +// // LSTM layer. +// LSTM<> lstm(inputSize, outputSize, rho); +// lstm.Reset(); +// lstm.ResetCell(rho); + +// // Initialize the weights to all ones. +// lstm.Parameters().ones(); + +// arma::mat inputWeight = arma::ones(outputSize, inputSize); +// arma::mat outputWeight = arma::ones(outputSize, outputSize); +// arma::mat bias = arma::ones(outputSize, input.n_cols); +// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); + +// for (size_t seqNum = 0; seqNum < rho; ++seqNum) +// { +// // Wrap a matrix around our data to avoid a copy. +// arma::mat stepData(input.slice(seqNum).memptr(), +// input.n_rows, input.n_cols, false, true); + +// if (cellLstm.is_empty()) +// { +// // Set the cell state to zeros. +// cellLstm = arma::zeros(outputSize, input.n_cols); +// cellCalc = arma::zeros(outputSize, input.n_cols); +// } +// else +// { +// // Set the cell state to zeros. +// cellLstm = arma::zeros(cellLstm.n_rows, cellLstm.n_cols); +// cellCalc = arma::zeros(cellCalc.n_rows, cellCalc.n_cols); +// } + +// // Apply Forward() on the LSTM layer. +// lstm.Forward(stepData, // Input. +// outLstm, // Output. +// cellLstm, // Cell state. +// true); // Write into cell state. + +// // Compute the value of cell state and output. +// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); + +// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); + +// // z = tanh(W.dot(x) + W.dot(h) + b). +// hidden = arma::tanh(inputWeight * stepData + +// outputWeight * outCalc + bias); + +// // c = f * c + i * z. +// cellCalc = forgetGate % cellCalc + inputGate % hidden; + +// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); + +// // h = o * tanh(c). +// outCalc = outputGate % arma::tanh(cellCalc); + +// CheckMatrices(outLstm, outCalc, 1e-12); +// CheckMatrices(cellLstm, cellCalc, 1e-12); +// } + +// // Attempting to write empty matrix into cell state. +// lstm.Reset(); +// lstm.ResetCell(rho); +// arma::mat stepData(input.slice(0).memptr(), +// input.n_rows, input.n_cols, false, true); + +// lstm.Forward(stepData, // Input. +// outLstm, // Output. +// cellLstm, // Cell state. +// true); // Write into cell state. + +// for (size_t seqNum = 1; seqNum < rho; ++seqNum) +// { +// arma::mat empty; +// // Should throw error. +// REQUIRE_THROWS_AS(lstm.Forward(stepData, // Input. +// outLstm, // Output. +// empty, // Cell state. +// true), // Write into cell state. +// std::runtime_error); +// } +// } + +// /** +// * Test that the functions that can modify and access the parameters of the +// * GRU layer work. +// */ +// TEST_CASE("GRULayerParametersTest", "[ANNLayerTest]") +// { +// // Parameter order : inSize, outSize, rho. +// GRU<> layer1(1, 2, 3); +// GRU<> layer2(1, 2, 4); + +// // Make sure we can get the parameters successfully. +// REQUIRE(layer1.InSize() == 1); +// REQUIRE(layer1.OutSize() == 2); +// REQUIRE(layer1.Rho() == 3); + +// // Now modify the parameters to match the second layer. +// layer1.Rho() = 4; + +// // Now ensure all the results are the same. +// REQUIRE(layer1.InSize() == layer2.InSize()); +// REQUIRE(layer1.OutSize() == layer2.OutSize()); +// REQUIRE(layer1.Rho() == layer2.Rho()); +// } + +// /** +// * Check if the gradients computed by GRU cell are close enough to the +// * approximation of the gradients. +// */ +// TEST_CASE("GradientGRULayerTest", "[ANNLayerTest]") +// { +// // GRU function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(1, 1, 5)), +// target(arma::ones(1, 1, 5)) +// { +// const size_t rho = 5; + +// model = new RNN(rho); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(1, 10); +// model->Add >(10, 3, rho); +// model->Add >(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// arma::mat output; +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// RNN* model; +// arma::cube input, target; +// } function; + +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +// /** +// * GRU layer manual forward test. +// */ +// TEST_CASE("ForwardGRULayerTest", "[ANNLayerTest]") +// { +// // This will make it easier to clean memory later. +// GRU<>* gruAlloc = new GRU<>(3, 3, 5); +// GRU<>& gru = *gruAlloc; + +// // Initialize the weights to all ones. +// NetworkInitialization +// networkInit(ConstInitialization(1)); +// networkInit.Initialize(gru.Model(), gru.Parameters()); + +// // Provide input of all ones. +// arma::mat input = arma::ones(3, 1); +// arma::mat output; + +// gru.Forward(input, output); + +// // Compute the z_t gate output. +// arma::mat expectedOutput = arma::ones(3, 1); +// expectedOutput *= -4; +// expectedOutput = arma::exp(expectedOutput); +// expectedOutput = arma::ones(3, 1) / (arma::ones(3, 1) + expectedOutput); +// expectedOutput = (arma::ones(3, 1) - expectedOutput) % expectedOutput; + +// // For the first input the output should be equal to the output of +// // gate z_t as the previous output fed to the cell is all zeros. +// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); + +// expectedOutput = output; + +// gru.Forward(input, output); + +// double s = arma::as_scalar(arma::sum(expectedOutput)); + +// // Compute the value of z_t gate for the second input. +// arma::mat z_t = arma::ones(3, 1); +// z_t *= -(s + 4); +// z_t = arma::exp(z_t); +// z_t = arma::ones(3, 1) / (arma::ones(3, 1) + z_t); + +// // Compute the value of o_t gate for the second input. +// arma::mat o_t = arma::ones(3, 1); +// o_t *= -(arma::as_scalar(arma::sum(expectedOutput % z_t)) + 4); +// o_t = arma::exp(o_t); +// o_t = arma::ones(3, 1) / (arma::ones(3, 1) + o_t); + +// // Expected output for the second input. +// expectedOutput = z_t % expectedOutput + (arma::ones(3, 1) - z_t) % o_t; + +// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); + +// LayerTypes<> layer(gruAlloc); +// boost::apply_visitor(DeleteVisitor(), layer); +// } + +/** + * Simple add merge module test. + */ +// TEST_CASE("SimpleAddMergeLayerTest", "[ANNLayerTest]") +// { +// arma::mat output, input, delta; +// input = arma::ones(10, 1); +// +// for (size_t i = 0; i < 5; ++i) +// { +// AddMerge<> module(false, false); +// const size_t numMergeModules = math::RandInt(2, 10); +// for (size_t m = 0; m < numMergeModules; ++m) +// { +// IdentityLayer<> identityLayer; +// identityLayer.Forward(input, identityLayer.OutputParameter()); +// +// module.Add >(identityLayer); +// } +// +// // Test the Forward function. +// module.Forward(input, output); +// REQUIRE(10 * numMergeModules == arma::accu(output)); +// +// // Test the Backward function. +// module.Backward(input, output, delta); +// REQUIRE(arma::accu(output) == arma::accu(delta)); +// } +// } + +/** + * Test the LSTM layer with a user defined rho parameter and without. + */ +// TEST_CASE("LSTMRrhoTest", "[ANNLayerTest]") +// { +// const size_t rho = 5; +// arma::cube input = arma::randu(1, 1, 5); +// arma::cube target = arma::zeros(1, 1, 5); +// RandomInitialization init(0.5, 0.5); +// +// // Create model with user defined rho parameter. +// RNN modelA( +// rho, false, NegativeLogLikelihood(), init); +// modelA.Add >(); +// modelA.Add >(1, 10); +// +// // Use LSTM layer with rho. +// modelA.Add >(10, 3, rho); +// modelA.Add >(); +// +// // Create model without user defined rho parameter. +// RNN modelB( +// rho, false, NegativeLogLikelihood(), init); +// modelB.Add >(); +// modelB.Add >(1, 10); +// +// // Use LSTM layer with rho = MAXSIZE. +// modelB.Add >(10, 3); +// modelB.Add >(); +// +// ens::StandardSGD opt(0.1, 1, 5, -100, false); +// modelA.Train(input, target, opt); +// modelB.Train(input, target, opt); +// +// CheckMatrices(modelB.Parameters(), modelA.Parameters()); +// } + +/** + * LSTM layer numerical gradient test. + */ +// TEST_CASE("GradientLSTMLayerTest", "[ANNLayerTest]") +// { +// // LSTM function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(1, 1, 5)), +// target(arma::zeros(1, 1, 5)) +// { +// const size_t rho = 5; +// +// model = new RNN(rho); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(1, 10); +// model->Add >(10, 3, rho); +// model->Add >(); +// } +// +// ~GradientFunction() +// { +// delete model; +// } +// +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } +// +// arma::mat& Parameters() { return model->Parameters(); } +// +// RNN* model; +// arma::cube input, target; +// } function; +// +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +/** + * Test that the functions that can modify and access the parameters of the + * LSTM layer work. + */ +// TEST_CASE("LSTMLayerParametersTest", "[ANNLayerTest]") +// { +// // Parameter order : inSize, outSize, rho. +// LSTM<> layer1(1, 2, 3); +// LSTM<> layer2(1, 2, 4); +// +// // Make sure we can get the parameters successfully. +// REQUIRE(layer1.InSize() == 1); +// REQUIRE(layer1.OutSize() == 2); +// REQUIRE(layer1.Rho() == 3); +// +// // Now modify the parameters to match the second layer. +// layer1.Rho() = 4; +// +// // Now ensure all the results are the same. +// REQUIRE(layer1.InSize() == layer2.InSize()); +// REQUIRE(layer1.OutSize() == layer2.OutSize()); +// REQUIRE(layer1.Rho() == layer2.Rho()); +// } + +/** + * Test the FastLSTM layer with a user defined rho parameter and without. + */ +// TEST_CASE("FastLSTMRrhoTest", "[ANNLayerTest]") +// { +// const size_t rho = 5; +// arma::cube input = arma::randu(1, 1, 5); +// arma::cube target = arma::zeros(1, 1, 5); +// RandomInitialization init(0.5, 0.5); +// +// // Create model with user defined rho parameter. +// RNN modelA( +// rho, false, NegativeLogLikelihood(), init); +// modelA.Add >(); +// modelA.Add >(1, 10); +// +// // Use FastLSTM layer with rho. +// modelA.Add >(10, 3, rho); +// modelA.Add >(); +// +// // Create model without user defined rho parameter. +// RNN modelB( +// rho, false, NegativeLogLikelihood(), init); +// modelB.Add >(); +// modelB.Add >(1, 10); +// +// // Use FastLSTM layer with rho = MAXSIZE. +// modelB.Add >(10, 3); +// modelB.Add >(); +// +// ens::StandardSGD opt(0.1, 1, 5, -100, false); +// modelA.Train(input, target, opt); +// modelB.Train(input, target, opt); +// +// CheckMatrices(modelB.Parameters(), modelA.Parameters()); +// } + +/** + * FastLSTM layer numerical gradient test. + */ +// TEST_CASE("GradientFastLSTMLayerTest", "[ANNLayerTest]") +// { +// // Fast LSTM function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(1, 1, 5)), +// target(arma::zeros(1, 1, 5)) +// { +// const size_t rho = 5; +// +// model = new RNN(rho); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(1, 10); +// model->Add >(10, 3, rho); +// model->Add >(); +// } +// +// ~GradientFunction() +// { +// delete model; +// } +// +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } +// +// arma::mat& Parameters() { return model->Parameters(); } +// +// RNN* model; +// arma::cube input, target; +// } function; +// +// // The threshold should be << 0.1 but since the Fast LSTM layer uses an +// // approximation of the sigmoid function the estimated gradient is not +// // correct. +// REQUIRE(CheckGradient(function) <= 0.2); +// } + +/** + * Test that the functions that can modify and access the parameters of the + * Fast LSTM layer work. + */ +// TEST_CASE("FastLSTMLayerParametersTest", "[ANNLayerTest]") +// { +// // Parameter order : inSize, outSize, rho. +// FastLSTM<> layer1(1, 2, 3); +// FastLSTM<> layer2(1, 2, 4); +// +// // Make sure we can get the parameters successfully. +// REQUIRE(layer1.InSize() == 1); +// REQUIRE(layer1.OutSize() == 2); +// REQUIRE(layer1.Rho() == 3); +// +// // Now modify the parameters to match the second layer. +// layer1.Rho() = 4; +// +// // Now ensure all the results are the same. +// REQUIRE(layer1.InSize() == layer2.InSize()); +// REQUIRE(layer1.OutSize() == layer2.OutSize()); +// REQUIRE(layer1.Rho() == layer2.Rho()); +// } + +/** + * Check whether copying and moving network with FastLSTM is working or not. + */ +// TEST_CASE("CheckCopyMoveFastLSTMTest", "[ANNLayerTest]") +// { +// arma::cube input = arma::randu(1, 1, 5); +// arma::cube target = arma::ones(1, 1, 5); +// const size_t rho = 5; +// +// RNN *model1 = +// new RNN(rho); +// model1->ResetData(input, target); +// model1->Add >(); +// model1->Add >(1, 10); +// model1->Add >(10, 3, rho); +// model1->Add >(); +// +// RNN *model2 = +// new RNN(rho); +// model2->ResetData(input, target); +// model2->Add >(); +// model2->Add >(1, 10); +// model2->Add >(10, 3, rho); +// model2->Add >(); +// +// // Check whether copy constructor is working or not. +// CheckRNNCopyFunction<>(model1, input, target, 1); +// +// // Check whether move constructor is working or not. +// CheckRNNMoveFunction<>(model2, input, target, 1); +// } + +/** + * Check whether copying and moving network with LSTM is working or not. + */ +// TEST_CASE("CheckCopyMoveLSTMTest", "[ANNLayerTest]") +// { +// arma::cube input = arma::randu(1, 1, 5); +// arma::cube target = arma::ones(1, 1, 5); +// const size_t rho = 5; +// +// RNN *model1 = +// new RNN(rho); +// model1->ResetData(input, target); +// model1->Add >(); +// model1->Add >(1, 10); +// model1->Add >(10, 3, rho); +// model1->Add >(); +// +// RNN *model2 = +// new RNN(rho); +// model2->ResetData(input, target); +// model2->Add >(); +// model2->Add >(1, 10); +// model2->Add >(10, 3, rho); +// model2->Add >(); +// +// // Check whether copy constructor is working or not. +// CheckRNNCopyFunction<>(model1, input, target, 1); +// +// // Check whether move constructor is working or not. +// CheckRNNMoveFunction<>(model2, input, target, 1); +// } + +/** + * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell + * state. Besides output, the overloaded function provides read access to cell + * state of the LSTM layer. + */ +// TEST_CASE("ReadCellStateParamLSTMLayerTest", "[ANNLayerTest]") +// { +// const size_t rho = 5, inputSize = 3, outputSize = 2; +// +// // Provide input of all ones. +// arma::cube input = arma::ones(inputSize, outputSize, rho); +// +// arma::mat inputGate, forgetGate, outputGate, hidden; +// arma::mat outLstm, cellLstm; +// +// // LSTM layer. +// LSTM<> lstm(inputSize, outputSize, rho); +// lstm.Reset(); +// lstm.ResetCell(rho); +// +// // Initialize the weights to all ones. +// lstm.Parameters().ones(); +// +// arma::mat inputWeight = arma::ones(outputSize, inputSize); +// arma::mat outputWeight = arma::ones(outputSize, outputSize); +// arma::mat bias = arma::ones(outputSize, input.n_cols); +// arma::mat cellCalc = arma::zeros(outputSize, input.n_cols); +// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); +// +// for (size_t seqNum = 0; seqNum < rho; ++seqNum) +// { +// // Wrap a matrix around our data to avoid a copy. +// arma::mat stepData(input.slice(seqNum).memptr(), +// input.n_rows, input.n_cols, false, true); +// +// // Apply Forward() on LSTM layer. +// lstm.Forward(stepData, // Input. +// outLstm, // Output. +// cellLstm, // Cell state. +// false); // Don't write into the cell state. +// +// // Compute the value of cell state and output. +// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); +// +// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); +// +// // z = tanh(W.dot(x) + W.dot(h) + b). +// hidden = arma::tanh(inputWeight * stepData + +// outputWeight * outCalc + bias); +// +// // c = f * c + i * z. +// cellCalc = forgetGate % cellCalc + inputGate % hidden; +// +// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); +// +// // h = o * tanh(c). +// outCalc = outputGate % arma::tanh(cellCalc); +// +// CheckMatrices(outLstm, outCalc, 1e-12); +// CheckMatrices(cellLstm, cellCalc, 1e-12); +// } +// } + +/** + * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell + * state. Besides output, the overloaded function provides write access to cell + * state of the LSTM layer. + */ +// TEST_CASE("WriteCellStateParamLSTMLayerTest", "[ANNLayerTest]") +// { +// const size_t rho = 5, inputSize = 3, outputSize = 2; +// +// // Provide input of all ones. +// arma::cube input = arma::ones(inputSize, outputSize, rho); +// +// arma::mat inputGate, forgetGate, outputGate, hidden; +// arma::mat outLstm, cellLstm; +// arma::mat cellCalc; +// +// // LSTM layer. +// LSTM<> lstm(inputSize, outputSize, rho); +// lstm.Reset(); +// lstm.ResetCell(rho); +// +// // Initialize the weights to all ones. +// lstm.Parameters().ones(); +// +// arma::mat inputWeight = arma::ones(outputSize, inputSize); +// arma::mat outputWeight = arma::ones(outputSize, outputSize); +// arma::mat bias = arma::ones(outputSize, input.n_cols); +// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); +// +// for (size_t seqNum = 0; seqNum < rho; ++seqNum) +// { +// // Wrap a matrix around our data to avoid a copy. +// arma::mat stepData(input.slice(seqNum).memptr(), +// input.n_rows, input.n_cols, false, true); +// +// if (cellLstm.is_empty()) +// { +// // Set the cell state to zeros. +// cellLstm = arma::zeros(outputSize, input.n_cols); +// cellCalc = arma::zeros(outputSize, input.n_cols); +// } +// else +// { +// // Set the cell state to zeros. +// cellLstm = arma::zeros(cellLstm.n_rows, cellLstm.n_cols); +// cellCalc = arma::zeros(cellCalc.n_rows, cellCalc.n_cols); +// } +// +// // Apply Forward() on the LSTM layer. +// lstm.Forward(stepData, // Input. +// outLstm, // Output. +// cellLstm, // Cell state. +// true); // Write into cell state. +// +// // Compute the value of cell state and output. +// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); +// +// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); +// +// // z = tanh(W.dot(x) + W.dot(h) + b). +// hidden = arma::tanh(inputWeight * stepData + +// outputWeight * outCalc + bias); +// +// // c = f * c + i * z. +// cellCalc = forgetGate % cellCalc + inputGate % hidden; +// +// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). +// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + +// outputWeight * outCalc + outputWeight % cellCalc + bias))); +// +// // h = o * tanh(c). +// outCalc = outputGate % arma::tanh(cellCalc); +// +// CheckMatrices(outLstm, outCalc, 1e-12); +// CheckMatrices(cellLstm, cellCalc, 1e-12); +// } +// +// // Attempting to write empty matrix into cell state. +// lstm.Reset(); +// lstm.ResetCell(rho); +// arma::mat stepData(input.slice(0).memptr(), +// input.n_rows, input.n_cols, false, true); +// +// lstm.Forward(stepData, // Input. +// outLstm, // Output. +// cellLstm, // Cell state. +// true); // Write into cell state. +// +// for (size_t seqNum = 1; seqNum < rho; ++seqNum) +// { +// arma::mat empty; +// // Should throw error. +// REQUIRE_THROWS_AS(lstm.Forward(stepData, // Input. +// outLstm, // Output. +// empty, // Cell state. +// true), // Write into cell state. +// std::runtime_error); +// } +// } + +/** + * Test that the functions that can modify and access the parameters of the + * GRU layer work. + */ +// TEST_CASE("GRULayerParametersTest", "[ANNLayerTest]") +// { +// // Parameter order : inSize, outSize, rho. +// GRU<> layer1(1, 2, 3); +// GRU<> layer2(1, 2, 4); +// +// // Make sure we can get the parameters successfully. +// REQUIRE(layer1.InSize() == 1); +// REQUIRE(layer1.OutSize() == 2); +// REQUIRE(layer1.Rho() == 3); +// +// // Now modify the parameters to match the second layer. +// layer1.Rho() = 4; +// +// // Now ensure all the results are the same. +// REQUIRE(layer1.InSize() == layer2.InSize()); +// REQUIRE(layer1.OutSize() == layer2.OutSize()); +// REQUIRE(layer1.Rho() == layer2.Rho()); +// } + +/** + * Check if the gradients computed by GRU cell are close enough to the + * approximation of the gradients. + */ +// TEST_CASE("GradientGRULayerTest", "[ANNLayerTest]") +// { +// // GRU function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(1, 1, 5)), +// target(arma::zeros(1, 1, 5)) +// { +// const size_t rho = 5; +// +// model = new RNN(rho); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(1, 10); +// model->Add >(10, 3, rho); +// model->Add >(); +// } +// +// ~GradientFunction() +// { +// delete model; +// } +// +// double Gradient(arma::mat& gradient) const +// { +// arma::mat output; +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } +// +// arma::mat& Parameters() { return model->Parameters(); } +// +// RNN* model; +// arma::cube input, target; +// } function; +// +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +/** + * GRU layer manual forward test. + */ +// TEST_CASE("ForwardGRULayerTest", "[ANNLayerTest]") +// { +// // This will make it easier to clean memory later. +// GRU<>* gruAlloc = new GRU<>(3, 3, 5); +// GRU<>& gru = *gruAlloc; +// +// // Initialize the weights to all ones. +// NetworkInitialization +// networkInit(ConstInitialization(1)); +// networkInit.Initialize(gru.Model(), gru.Parameters()); +// +// // Provide input of all ones. +// arma::mat input = arma::ones(3, 1); +// arma::mat output; +// +// gru.Forward(input, output); +// +// // Compute the z_t gate output. +// arma::mat expectedOutput = arma::ones(3, 1); +// expectedOutput *= -4; +// expectedOutput = arma::exp(expectedOutput); +// expectedOutput = arma::ones(3, 1) / (arma::ones(3, 1) + expectedOutput); +// expectedOutput = (arma::ones(3, 1) - expectedOutput) % expectedOutput; +// +// // For the first input the output should be equal to the output of +// // gate z_t as the previous output fed to the cell is all zeros. +// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); +// +// expectedOutput = output; +// +// gru.Forward(input, output); +// +// double s = arma::as_scalar(arma::sum(expectedOutput)); +// +// // Compute the value of z_t gate for the second input. +// arma::mat z_t = arma::ones(3, 1); +// z_t *= -(s + 4); +// z_t = arma::exp(z_t); +// z_t = arma::ones(3, 1) / (arma::ones(3, 1) + z_t); +// +// // Compute the value of o_t gate for the second input. +// arma::mat o_t = arma::ones(3, 1); +// o_t *= -(arma::as_scalar(arma::sum(expectedOutput % z_t)) + 4); +// o_t = arma::exp(o_t); +// o_t = arma::ones(3, 1) / (arma::ones(3, 1) + o_t); +// +// // Expected output for the second input. +// expectedOutput = z_t % expectedOutput + (arma::ones(3, 1) - z_t) % o_t; +// +// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); +// +// LayerTypes<> layer(gruAlloc); +// boost::apply_visitor(DeleteVisitor(), layer); +// } + +/** + * Simple concat module test. + * +TEST_CASE("SimpleConcatLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta, error; + + Linear* moduleA = new Linear(10, 10); + moduleA->Parameters().randu(); + moduleA->Reset(); + + Linear* moduleB = new Linear(10, 10); + moduleB->Parameters().randu(); + moduleB->Reset(); + + Concat module; + module.Add(moduleA); + module.Add(moduleB); + + // Test the Forward function. + input = arma::zeros(10, 1); + module.Forward(input, output); + + const double sumModuleA = arma::accu( + moduleA->Parameters().submat( + 100, 0, moduleA->Parameters().n_elem - 1, 0)); + const double sumModuleB = arma::accu( + moduleB->Parameters().submat( + 100, 0, moduleB->Parameters().n_elem - 1, 0)); + REQUIRE(sumModuleA + sumModuleB == + Approx(arma::accu(output.col(0))).epsilon(1e-5)); + + // Test the Backward function. + error = arma::zeros(20, 1); + module.Backward(input, error, delta); + REQUIRE(arma::accu(delta) == 0); +} +*/ + +/** + * Test to check Concat layer along different axes. + * +TEST_CASE("ConcatAlongAxisTest", "[ANNLayerTest]") +{ + arma::mat output, input, error, outputA, outputB; + size_t inputWidth = 4, inputHeight = 4, inputChannel = 2; + size_t outputWidth, outputHeight, outputChannel = 2; + size_t kW = 3, kH = 3; + size_t batch = 1; + + // Using Convolution<> layer as inout to Concat<> layer. + // Compute the output shape of convolution layer. + outputWidth = (inputWidth - kW) + 1; + outputHeight = (inputHeight - kH) + 1; + + input = arma::ones(inputWidth * inputHeight * inputChannel, batch); + + Convolution* moduleA = new Convolution(inputChannel, outputChannel, + kW, kH, 1, 1, 0, 0, inputWidth, inputHeight); + Convolution* moduleB = new Convolution(inputChannel, outputChannel, + kW, kH, 1, 1, 0, 0, inputWidth, inputHeight); + + moduleA->Reset(); + moduleA->Parameters().randu(); + moduleB->Reset(); + moduleB->Parameters().randu(); + + // Compute output of each layer. + moduleA->Forward(input, outputA); + moduleB->Forward(input, outputB); + + arma::cube A(outputA.memptr(), outputWidth, outputHeight, outputChannel); + arma::cube B(outputB.memptr(), outputWidth, outputHeight, outputChannel); + + error = arma::ones(outputWidth * outputHeight * outputChannel * 2, 1); + + for (size_t axis = 0; axis < 3; ++axis) + { + size_t x = 1, y = 1, z = 1; + arma::cube calculatedOut; + if (axis == 0) + { + calculatedOut.set_size(2 * outputWidth, outputHeight, outputChannel); + for (size_t i = 0; i < A.n_slices; ++i) + { + arma::mat aMat = A.slice(i); + arma::mat bMat = B.slice(i); + calculatedOut.slice(i) = arma::join_cols(aMat, bMat); + } + x = 2; + } + if (axis == 1) + { + calculatedOut.set_size(outputWidth, 2 * outputHeight, outputChannel); + for (size_t i = 0; i < A.n_slices; ++i) + { + arma::mat aMat = A.slice(i); + arma::mat bMat = B.slice(i); + calculatedOut.slice(i) = arma::join_rows(aMat, bMat); + } + y = 2; + } + if (axis == 2) + { + calculatedOut = arma::join_slices(A, B); + z = 2; + } + + // Compute output of Concat<> layer. + arma::Row inputSize{outputWidth, outputHeight, outputChannel}; + Concat module(inputSize, axis, true); + module.Add(moduleA); + module.Add(moduleB); + module.Forward(input, output); + arma::cube concatOut(output.memptr(), x * outputWidth, + y * outputHeight, z * outputChannel); + + // Verify if the output reshaped to cubes are similar. + CheckMatrices(concatOut, calculatedOut, 1e-12); + } + delete moduleA; + delete moduleB; +}*/ + +/** + * Test that the function that can access the axis parameter of the + * Concat layer works. + * +TEST_CASE("ConcatLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : inputSize{width, height, channels}, axis, model, run. + arma::Row inputSize{128, 128, 3}; + Concat layer(inputSize, 2, false, true); + + // Make sure we can get the parameters successfully. + REQUIRE(layer.ConcatAxis() == 2); +} +*/ + +/** + * Concat layer numerical gradient test. + */ +// TEST_CASE("GradientConcatLayerTest", "[ANNLayerTest]") +// { +// // Concat function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(10, 1)), +// target(arma::mat("0")) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add(); +// model->Add(10, 10); + +// concat = new Concat(true); +// concat->Add(10, 2); +// model->Add(concat); + +// model->Add(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// FFN* model; +// Concat* concat; +// arma::mat input, target; +// } function; + +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +/** + * Simple concatenate module test. + * +TEST_CASE("SimpleConcatenateLayerTest", "[ANNLayerTest]") +{ + arma::mat input = arma::ones(5, 1); + arma::mat output, delta; + + Concatenate module; + module.Concat() = arma::ones(5, 1) * 0.5; + + // Test the Forward function. + module.Forward(input, output); + + REQUIRE(arma::accu(output) == 7.5); + + // Test the Backward function. + module.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 5); +} +*/ + +/** + * Concatenate layer numerical gradient test. + * +TEST_CASE("GradientConcatenateLayerTest", "[ANNLayerTest]") +{ + // Concatenate function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(10, 1)), + target(arma::mat("0")) + { + model = new FFN(); + model->ResetData(input, target); + model->Add(); + model->Add(10, 5); + + arma::mat concat = arma::ones(5, 1); + // concatenate = new Concatenate(); + // concatenate->Concat() = concat; + // model->Add(concatenate); + model->Add(concat); + + model->Add(10, 5); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + Concatenate* concatenate; + arma::mat input, target; + } function; + + REQUIRE(CheckGradient(function) <= 1e-4); +} +*/ + +/** + * Simple lookup module test. + * +TEST_CASE("SimpleLookupLayerTest", "[ANNLayerTest]") +{ + const size_t vocabSize = 10; + const size_t embeddingSize = 2; + const size_t seqLength = 3; + const size_t batchSize = 4; + + arma::mat output, input, gy, g, gradient; + + Lookup module(vocabSize, embeddingSize); + module.Parameters().randu(); + + // Test the Forward function. + input = arma::zeros(seqLength, batchSize); + for (size_t i = 0; i < input.n_elem; ++i) + { + int token = math::RandInt(1, vocabSize); + input(i) = token; + } + + module.Forward(input, output); + for (size_t i = 0; i < batchSize; ++i) + { + // The Lookup module uses index - 1 for the cols. + const double outputSum = arma::accu(module.Parameters().cols( + arma::conv_to::from(input.col(i)) - 1)); + + REQUIRE(std::fabs(outputSum - arma::accu(output.col(i))) <= 1e-5); + } + + // Test the Gradient function. + arma::mat error = 0.01 * arma::randu(embeddingSize * seqLength, batchSize); + module.Gradient(input, error, gradient); + + REQUIRE(std::fabs(arma::accu(error) - arma::accu(gradient)) <= 1e-07); +} +*/ + +/** + * Lookup layer numerical gradient test. + * +TEST_CASE("GradientLookupLayerTest", "[ANNLayerTest]") +{ + // Lookup function gradient instantiation. + struct GradientFunction + { + GradientFunction() + { + input.set_size(seqLength, batchSize); + for (size_t i = 0; i < input.n_elem; ++i) + { + input(i) = math::RandInt(1, vocabSize); + } + target = arma::zeros(vocabSize, batchSize); + for (size_t i = 0; i < batchSize; ++i) + { + const size_t targetWord = math::RandInt(1, vocabSize); + target(targetWord, i) = 1; + } + + model = new FFN, GlorotInitialization>(BCELoss<>(1e-10, false)); + model->ResetData(input, target); + model->Add(vocabSize, embeddingSize); + model->Add(embeddingSize * seqLength, vocabSize); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, batchSize); + model->Gradient(model->Parameters(), 0, gradient, batchSize); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN, GlorotInitialization>* model; + arma::mat input, target; + + const size_t seqLength = 10; + const size_t embeddingSize = 8; + const size_t vocabSize = 20; + const size_t batchSize = 4; + } function; + + REQUIRE(CheckGradient(function) <= 1e-6); +} +*/ + +/** + * Test that the functions that can access the parameters of the + * Lookup layer work. + * +TEST_CASE("LookupLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : vocabSize, embedingSize. + Lookup layer(100, 8); + + // Make sure we can get the parameters successfully. + REQUIRE(layer.VocabSize() == 100); + REQUIRE(layer.EmbeddingSize() == 8); +} +*/ + +/** + * Simple Softmax module test. + * +TEST_CASE("SimpleSoftmaxLayerTest", "[ANNLayerTest]") +{ + arma::mat input, output, gy, g; + Softmax module; + + // Test the forward function. + input = arma::mat("1.7; 3.6"); + module.Forward(input, output); + REQUIRE(arma::accu(arma::abs(arma::mat("0.130108; 0.869892") - output)) == + Approx(0.0).margin(1e-4)); + + // Test the backward function. + gy = arma::zeros(input.n_rows, input.n_cols); + gy(0) = 1; + module.Backward(output, gy, g); + REQUIRE(arma::accu(arma::abs(arma::mat("0.11318; -0.11318") - g)) == + Approx(0.0).margin(1e-04)); +} +*/ + +/** + * Softmax layer numerical gradient test. + * +TEST_CASE("GradientSoftmaxTest", "[ANNLayerTest]") +{ + // Softmax function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(10, 1)), + target(arma::mat("1; 0")) + { + model = new FFN; + model->ResetData(input, target); + model->Add(10, 10); + model->Add(); + model->Add(10, 2); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + REQUIRE(CheckGradient(function) <= 1e-4); +} +*/ + +/** + * Simple test for the NearestInterpolation layer + * +TEST_CASE("SimpleNearestInterpolationLayerTest", "[ANNLayerTest]") +{ + // Tested output against torch.nn.Upsample(mode="nearest"). + arma::mat input, output, unzoomedOutput, expectedOutput; + size_t inRowSize = 2; + size_t inColSize = 2; + size_t outRowSize = 5; + size_t outColSize = 7; + size_t depth = 1; + input.zeros(inRowSize * inColSize * depth, 1); + input[0] = 1.0; + input[1] = 3.0; + input[2] = 2.0; + input[3] = 4.0; + NearestInterpolation<> layer(inRowSize, inColSize, outRowSize, + outColSize, depth); + + expectedOutput << 1.0000 << 1.0000 << 1.0000 << 1.0000 << 2.0000 + << 2.0000 << 2.0000 << arma::endr + << 1.0000 << 1.0000 << 1.0000 << 1.0000 << 2.0000 + << 2.0000 << 2.0000 << arma::endr + << 1.0000 << 1.0000 << 1.0000 << 1.0000 << 2.0000 + << 2.0000 << 2.0000 << arma::endr + << 3.0000 << 3.0000 << 3.0000 << 3.0000 << 4.0000 + << 4.0000 << 4.0000 << arma::endr + << 3.0000 << 3.0000 << 3.0000 << 3.0000 << 4.0000 + << 4.0000 << 4.0000 << arma::endr; + expectedOutput.reshape(35, 1); + + layer.Forward(input, output); + CheckMatrices(output - expectedOutput, + arma::zeros(output.n_rows), 1e-4); + + expectedOutput.clear(); + expectedOutput << 12.0000 << 18.0000 << arma::endr + << 24.0000 << 24.0000 << arma::endr; + expectedOutput.reshape(4, 1); + layer.Backward(output, output, unzoomedOutput); + CheckMatrices(unzoomedOutput - expectedOutput, + arma::zeros(input.n_rows), 1e-4); + + arma::mat input1, output1, unzoomedOutput1, expectedOutput1; + inRowSize = 2; + inColSize = 3; + outRowSize = 17; + outColSize = 23; + input1 << 1 << 2 << 3 << arma::endr + << 4 << 5 << 6 << arma::endr; + input1.reshape(6, 1); + NearestInterpolation<> layer1(inRowSize, inColSize, outRowSize, + outColSize, depth); + + layer1.Forward(input1, output1); + layer1.Backward(output1, output1, unzoomedOutput1); + + REQUIRE(arma::accu(output1) - 1317.00 == Approx(0.0).margin(1e-05)); + REQUIRE(arma::accu(unzoomedOutput1) - 1317.00 == + Approx(0.0).margin(1e-05)); +} +*/ + +/* + * Simple test for the BilinearInterpolation layer + * +TEST_CASE("SimpleBilinearInterpolationLayerTest", "[ANNLayerTest]") +{ + // Tested output against tensorflow.image.resize_bilinear() + arma::mat input, output, unzoomedOutput, expectedOutput; + size_t inRowSize = 2; + size_t inColSize = 2; + size_t outRowSize = 5; + size_t outColSize = 5; + size_t depth = 1; + input.zeros(inRowSize * inColSize * depth, 1); + input[0] = 1.0; + input[1] = input[2] = 2.0; + input[3] = 3.0; + BilinearInterpolation layer(inRowSize, inColSize, outRowSize, outColSize, + depth); + expectedOutput = arma::mat("1.0000 1.4000 1.8000 2.0000 2.0000 \ + 1.4000 1.8000 2.2000 2.4000 2.4000 \ + 1.8000 2.2000 2.6000 2.8000 2.8000 \ + 2.0000 2.4000 2.8000 3.0000 3.0000 \ + 2.0000 2.4000 2.8000 3.0000 3.0000"); + expectedOutput.reshape(25, 1); + layer.Forward(input, output); + CheckMatrices(output - expectedOutput, arma::zeros(output.n_rows), 1e-12); + + expectedOutput = arma::mat("1.0000 1.9000 1.9000 2.8000"); + expectedOutput.reshape(4, 1); + layer.Backward(output, output, unzoomedOutput); + CheckMatrices(unzoomedOutput - expectedOutput, + arma::zeros(input.n_rows), 1e-12); +} +*/ + +/** + * Test that the functions that can modify and access the parameters of the + * Bilinear Interpolation layer work. + * +TEST_CASE("BilinearInterpolationLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : inRowSize, inColSize, outRowSize, outColSize, depth. + BilinearInterpolation layer1(1, 2, 3, 4, 5); + BilinearInterpolation layer2(2, 3, 4, 5, 6); + + // Make sure we can get the parameters successfully. + REQUIRE(layer1.InRowSize() == 1); + REQUIRE(layer1.InColSize() == 2); + REQUIRE(layer1.OutRowSize() == 3); + REQUIRE(layer1.OutColSize() == 4); + REQUIRE(layer1.InDepth() == 5); + + // Now modify the parameters to match the second layer. + layer1.InRowSize() = 2; + layer1.InColSize() = 3; + layer1.OutRowSize() = 4; + layer1.OutColSize() = 5; + layer1.InDepth() = 6; + + // Now ensure all results are the same. + REQUIRE(layer1.InRowSize() == layer2.InRowSize()); + REQUIRE(layer1.InColSize() == layer2.InColSize()); + REQUIRE(layer1.OutRowSize() == layer2.OutRowSize()); + REQUIRE(layer1.OutColSize() == layer2.OutColSize()); + REQUIRE(layer1.InDepth() == layer2.InDepth()); +} +*/ + +/* + * Simple test for the BicubicInterpolation layer. + * +TEST_CASE("SimpleBicubicInterpolationLayerTest", "[ANNLayerTest]") +{ + // Tested output against torch.nn.Upsample(mode="bicubic"). + // Test case with square input with rectangular output. + arma::mat input, output, unzoomedOutput, expectedOutput; + size_t inRowSize = 2; + size_t inColSize = 2; + size_t outRowSize = 5; + size_t outColSize = 7; + size_t depth = 1; + input.zeros(inRowSize * inColSize * depth, 1); + + input << 10 << 20 << arma::endr + << 30 << 40 << arma::endr; + input.reshape(4, 1); + BicubicInterpolation<> layer(inRowSize, inColSize, outRowSize, + outColSize, depth); + + expectedOutput << 6.68803935860 << 7.33308309038 << 9.69733236152 + << 12.79500000000 << 15.89266763848 << 18.25691690962 + << 18.90196064140 << arma::endr + << 10.53303935860 << 11.17808309038 << 13.54233236152 + << 16.64000000000 << 19.73766763848 << 22.10191690962 + << 22.74696064140 << arma::endr + << 18.89303935860 << 19.53808309038 << 21.90233236152 + << 25.00000000000 << 28.09766763848 << 30.46191690962 + << 31.10696064140 << arma::endr + << 27.25303935860 << 27.89808309038 << 30.26233236152 + << 33.36000000000 << 36.45766763848 << 38.82191690962 + << 39.46696064140 << arma::endr + << 31.09803935860 << 31.74308309038 << 34.10733236152 + << 37.20500000000 << 40.30266763848 << 42.66691690962 + << 43.31196064140 << arma::endr; + expectedOutput.reshape(35, 1); + layer.Forward(input, output); + + CheckMatrices(output, expectedOutput, 1e-6); + + expectedOutput.clear(); + expectedOutput << 103.79040654914 << 180.51345595086 << arma::endr + << 256.98654404914 << 333.70959345086 << arma::endr; + expectedOutput.reshape(4, 1); + + layer.Backward(output, output, unzoomedOutput); + + CheckMatrices(unzoomedOutput, expectedOutput, 1e-6); + + // Tested output against torch.nn.Upsample(mode="bicubic"). + // Test case with rectangular input with rectangular output. + arma::mat input1, output1, unzoomedOutput1, expectedOutput1, expectedUnzoomed; + + inRowSize = 2; + inColSize = 3; + outRowSize = 5; + outColSize = 7; + depth = 1; + input1.zeros(inRowSize * inColSize * depth, 1); + + input1 << 10 << 20 << 30 << arma::endr + << 40 << 50 << 60 << arma::endr; + input1.reshape(6, 1); + + BicubicInterpolation<> layer1(inRowSize, inColSize, outRowSize, + outColSize, depth); + + expectedOutput1 << 5.59920553936 << 7.77121720117 << 11.44468658892 + << 16.69250000000 << 21.94031341108 << 25.61378279883 + << 27.78579446064 << arma::endr + << 11.36670553936 << 13.53871720117 << 17.21218658892 + << 22.46000000000 << 27.70781341108 << 31.38128279883 + << 33.55329446064 << arma::endr + << 23.90670553936 << 26.07871720117 << 29.75218658892 + << 35.00000000000 << 40.24781341108 << 43.92128279883 + << 46.09329446064 << arma::endr + << 36.44670553936 << 38.61871720117 << 42.29218658892 + << 47.54000000000 << 52.78781341108 << 56.46128279883 + << 58.63329446064 << arma::endr + << 42.21420553936 << 44.38621720117 << 48.05968658892 + << 53.30750000000 << 58.55531341108 << 62.22878279883 + << 64.40079446064 << arma::endr; + expectedOutput1.reshape(35, 1); + layer1.Forward(input1, output1); + + CheckMatrices(output1, expectedOutput1, 1e-6); + + expectedUnzoomed << 67.65674505130 << 132.29729646501 + << 182.75175223368 << arma::endr + << 218.01355388877 << 291.17209129009 + << 333.10856107115 << arma::endr; + expectedUnzoomed.reshape(6, 1); + + layer1.Backward(output1, output1, unzoomedOutput1); + CheckMatrices(unzoomedOutput1, expectedUnzoomed, 1e-6); +} +*/ + +// /** +// * Tests the BatchNorm Layer, compares the layers parameters with +// * the values from another implementation. +// * Link to the implementation - http://cthorey.github.io./backpropagation/ +// */ +// TEST_CASE("BatchNormTest", "[ANNLayerTest]") +// { +// arma::mat input, output; +// input << 5.1 << 3.5 << 1.4 << arma::endr +// << 4.9 << 3.0 << 1.4 << arma::endr +// << 4.7 << 3.2 << 1.3 << arma::endr; + +// // BatchNorm layer with average parameter set to true. +// BatchNorm<> model(input.n_rows); +// model.Reset(); + +// // BatchNorm layer with average parameter set to false. +// BatchNorm<> model2(input.n_rows, 1e-5, false); +// model2.Reset(); + +// // Non-Deteministic Forward Pass Test. +// model.Deterministic() = false; +// model.Forward(input, output); + +// // Value calculates using torch.nn.BatchNorm2d(momentum = None). +// arma::mat result; +// result = { { 1.1658, 0.1100, -1.2758 }, +// { 1.2579, -0.0699, -1.1880}, +// { 1.1737, 0.0958, -1.2695 } }; + +// CheckMatrices(output, result, 1e-1); + +// model2.Forward(input, output); +// CheckMatrices(output, result, 1e-1); +// result.clear(); + +// // Values calculated using torch.nn.BatchNorm2d(momentum = None). +// output = model.TrainingMean(); +// result = arma::mat({ 3.33333333, 3.1, 3.06666666 }).t(); + +// CheckMatrices(output, result, 1e-1); + +// // Values calculated using torch.nn.BatchNorm2d(). +// output = model2.TrainingMean(); +// result = arma::mat({ 0.3333, 0.3100, 0.3067 }).t(); + +// CheckMatrices(output, result, 1e-1); +// result.clear(); + + // Values calculated using torch.nn.BatchNorm2d(momentum = None). +// output = model.TrainingVariance(); +// result = arma::mat({ 3.4433, 3.0700, 2.9033 }).t(); + +// CheckMatrices(output, result, 1e-1); +// result.clear(); + + // Values calculated using torch.nn.BatchNorm2d(). +// output = model2.TrainingVariance(); +// result = arma::mat({ 1.2443, 1.2070, 1.1903 }).t(); + +// CheckMatrices(output, result, 1e-1); +// result.clear(); + +// // Deterministic Forward Pass test. +// model.Deterministic() = true; +// model.Forward(input, output); + + // Values calculated using torch.nn.BatchNorm2d(momentum = None). +// result = { { 0.9521, 0.0898, -1.0419 }, +// { 1.0273, -0.0571, -0.9702 }, +// { 0.9586, 0.0783, -1.0368 } }; + +// CheckMatrices(output, result, 1e-1); + +// // Values calculated using torch.nn.BatchNorm2d(). +// model2.Deterministic() = true; +// model2.Forward(input, output); + +// result = { { 4.2731, 2.8388, 0.9562 }, +// { 4.1779, 2.4485, 0.9921 }, +// { 4.0268, 2.6519, 0.9105 } }; +// +// CheckMatrices(output, result, 1e-1); +// } + +// /** +// * BatchNorm layer numerical gradient test. +// */ +// TEST_CASE("GradientBatchNormTest", "[ANNLayerTest]") +// { +// bool pass = false; +// for (size_t trial = 0; trial < 10; trial++) +// { +// // Add function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randn(32, 2048)), +// target(arma::zeros(1, 2048)) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(32, 4); +// model->Add >(4); +// model->Add>(4, 2); +// model->Add >(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 2048, false); +// model->Gradient(model->Parameters(), 0, gradient, 2048); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// FFN* model; +// arma::mat input, target; +// } function; + +// double gradient = CheckGradient(function); +// if (gradient < 2e-1) +// { +// pass = true; +// break; +// } +// } + +// REQUIRE(pass); +// } + +// /** +// * Test that the functions that can access the parameters of the +// * Batch Norm layer work. +// */ +// TEST_CASE("BatchNormLayerParametersTest", "[ANNLayerTest]") +// { +// // Parameter order : size, eps. +// BatchNorm<> layer(7, 1e-3); + +// // Make sure we can get the parameters successfully. +// REQUIRE(layer.InputSize() == 7); +// REQUIRE(layer.Epsilon() == 1e-3); + +// arma::mat runningMean(7, 1, arma::fill::randn); +// arma::mat runningVariance(7, 1, arma::fill::randn); + +// layer.TrainingVariance() = runningVariance; +// layer.TrainingMean() = runningMean; +// CheckMatrices(layer.TrainingVariance(), runningVariance); +// CheckMatrices(layer.TrainingMean(), runningMean); +// } + +/** + * VirtualBatchNorm layer numerical gradient test. + * +TEST_CASE("GradientVirtualBatchNormTest", "[ANNLayerTest]") +{ + // Add function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randn(5, 256)), + target(arma::zeros(1, 256)) + { + arma::mat referenceBatch = arma::mat(input.memptr(), input.n_rows, 4); + + model = new FFN(); + model->ResetData(input, target); + model->Add(); + model->Add(5, 5); + model->Add(referenceBatch, 5); + model->Add(5, 2); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 16, false); + model->Gradient(model->Parameters(), 0, gradient, 16); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + REQUIRE(CheckGradient(function) <= 1e-4); +} +*/ + +/** + * Test that the functions that can modify and access the parameters of the + * Virtual Batch Norm layer work. + * +TEST_CASE("VirtualBatchNormLayerParametersTest", "[ANNLayerTest]") +{ + arma::mat input = arma::randn(5, 16); + arma::mat referenceBatch = arma::mat(input.memptr(), input.n_rows, 4); + + // Parameter order : referenceBatch, size, eps. + VirtualBatchNorm layer(referenceBatch, 5, 1e-3); + + // Make sure we can get the parameters successfully. + REQUIRE(layer.InSize() == 5); + REQUIRE(layer.Epsilon() == 1e-3); +} +*/ + +// /** +// * MiniBatchDiscrimination layer numerical gradient test. +// */ +// TEST_CASE("MiniBatchDiscriminationTest", "[ANNLayerTest]") +// { +// // Add function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randn(5, 4)), +// target(arma::zeros(1, 4)) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(5, 5); +// model->Add >(5, 10, 16); +// model->Add >(10, 2); +// model->Add >(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// return model->EvaluateWithGradient(model->Parameters(), 0, gradient, 4); +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// FFN* model; +// arma::mat input, target; +// } function; + +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +/** + * Simple Transposed Convolution layer test. + * +TEST_CASE("SimpleTransposedConvolutionLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + + TransposedConvolution module1(1, 1, 3, 3, 1, 1, 0, 0, 4, 4, 6, 6); + // Test the forward function. + input = arma::linspace(0, 15, 16); + module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module1.Parameters()(0) = 1.0; + module1.Parameters()(8) = 2.0; + module1.Reset(); + module1.Forward(input, output); + // Value calculated using tensorflow.nn.conv2d_transpose() + REQUIRE(arma::accu(output) == 360.0); + + // Test the backward function. + module1.Backward(input, output, delta); + // Value calculated using tensorflow.nn.conv2d() + REQUIRE(arma::accu(delta) == 720.0); + + TransposedConvolution module2(1, 1, 4, 4, 1, 1, 1, 1, 5, 5, 6, 6); + // Test the forward function. + input = arma::linspace(0, 24, 25); + module2.Parameters() = arma::mat(16 + 1, 1, arma::fill::zeros); + module2.Parameters()(0) = 1.0; + module2.Parameters()(3) = 1.0; + module2.Parameters()(6) = 1.0; + module2.Parameters()(9) = 1.0; + module2.Parameters()(12) = 1.0; + module2.Parameters()(15) = 2.0; + module2.Reset(); + module2.Forward(input, output); + // Value calculated using torch.nn.functional.conv_transpose2d() + REQUIRE(arma::accu(output) == 1512.0); + + // Test the backward function. + module2.Backward(input, output, delta); + // Value calculated using torch.nn.functional.conv2d() + REQUIRE(arma::accu(delta) == 6504.0); + + TransposedConvolution module3(1, 1, 3, 3, 1, 1, 1, 1, 5, 5, 5, 5); + // Test the forward function. + input = arma::linspace(0, 24, 25); + module3.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module3.Parameters()(1) = 2.0; + module3.Parameters()(2) = 4.0; + module3.Parameters()(3) = 3.0; + module3.Parameters()(8) = 1.0; + module3.Reset(); + module3.Forward(input, output); + // Value calculated using torch.nn.functional.conv_transpose2d() + REQUIRE(arma::accu(output) == 2370.0); + + // Test the backward function. + module3.Backward(input, output, delta); + // Value calculated using torch.nn.functional.conv2d() + REQUIRE(arma::accu(delta) == 19154.0); + + TransposedConvolution module4(1, 1, 3, 3, 1, 1, 0, 0, 5, 5, 7, 7); + // Test the forward function. + input = arma::linspace(0, 24, 25); + module4.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module4.Parameters()(2) = 2.0; + module4.Parameters()(4) = 4.0; + module4.Parameters()(6) = 6.0; + module4.Parameters()(8) = 8.0; + module4.Reset(); + module4.Forward(input, output); + // Value calculated using torch.nn.functional.conv_transpose2d() + REQUIRE(arma::accu(output) == 6000.0); + + // Test the backward function. + module4.Backward(input, output, delta); + // Value calculated using torch.nn.functional.conv2d() + REQUIRE(arma::accu(delta) == 86208.0); + + TransposedConvolution module5(1, 1, 3, 3, 2, 2, 0, 0, 2, 2, 5, 5); + // Test the forward function. + input = arma::linspace(0, 3, 4); + module5.Parameters() = arma::mat(25 + 1, 1, arma::fill::zeros); + module5.Parameters()(2) = 8.0; + module5.Parameters()(4) = 6.0; + module5.Parameters()(6) = 4.0; + module5.Parameters()(8) = 2.0; + module5.Reset(); + module5.Forward(input, output); + // Value calculated using torch.nn.functional.conv_transpose2d() + REQUIRE(arma::accu(output) == 120.0); + + // Test the backward function. + module5.Backward(input, output, delta); + // Value calculated using torch.nn.functional.conv2d() + REQUIRE(arma::accu(delta) == 960.0); + + TransposedConvolution module6(1, 1, 3, 3, 2, 2, 1, 1, 3, 3, 5, 5); + // Test the forward function. + input = arma::linspace(0, 8, 9); + module6.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module6.Parameters()(0) = 8.0; + module6.Parameters()(3) = 6.0; + module6.Parameters()(6) = 2.0; + module6.Parameters()(8) = 4.0; + module6.Reset(); + module6.Forward(input, output); + // Value calculated using torch.nn.functional.conv_transpose2d() + REQUIRE(arma::accu(output) == 410.0); + + // Test the backward function. + module6.Backward(input, output, delta); + // Value calculated using torch.nn.functional.conv2d() + REQUIRE(arma::accu(delta) == 4444.0); + + TransposedConvolution module7(1, 1, 3, 3, 2, 2, 1, 1, 3, 3, 6, 6); + // Test the forward function. + input = arma::linspace(0, 8, 9); + module7.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module7.Parameters()(0) = 8.0; + module7.Parameters()(2) = 6.0; + module7.Parameters()(4) = 2.0; + module7.Parameters()(8) = 4.0; + module7.Reset(); + module7.Forward(input, output); + // Value calculated using torch.nn.functional.conv_transpose2d() + REQUIRE(arma::accu(output) == 606.0); + + module7.Backward(input, output, delta); + // Value calculated using torch.nn.functional.conv2d() + REQUIRE(arma::accu(delta) == 7732.0); +} +*/ + +/** + * Transposed Convolution layer numerical gradient test. + * +TEST_CASE("GradientTransposedConvolutionLayerTest", "[ANNLayerTest]") +{ + // Add function gradient instantiation. + // To make this test robust, check it five times. + bool pass = false; + for (size_t trial = 0; trial < 5; trial++) + { + struct GradientFunction + { + GradientFunction() : + input(arma::linspace(0, 35, 36)), + target(arma::mat("0")) + { + model = new FFN(); + model->ResetData(input, target); + model->Add(1, 1, 3, 3, 2, 2, 1, 1, 6, 6, 12, 12); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + if (CheckGradient(function) < 1e-3) + { + pass = true; + break; + } + } + REQUIRE(pass == true); +} +*/ + +/** + * Simple MultiplyMerge module test. + * +TEST_CASE("SimpleMultiplyMergeLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + input = arma::ones(10, 1); + + for (size_t i = 0; i < 5; ++i) + { + MultiplyMerge module(false, false); + const size_t numMergeModules = math::RandInt(2, 10); + for (size_t m = 0; m < numMergeModules; ++m) + { + IdentityLayer* identityLayer = new IdentityLayer(); + identityLayer->Forward(input, identityLayer->OutputParameter()); + + module.Add(identityLayer); + } + + // Test the Forward function. + module.Forward(input, output); + REQUIRE(10 == arma::accu(output)); + + // Test the Backward function. + module.Backward(input, output, delta); + REQUIRE(arma::accu(output) == arma::accu(delta)); + } +} +*/ + +/** + * Check whether copying and moving network with MultiplyMerge is working or + * not. + */ +// TEST_CASE("CheckCopyMoveMultiplyMergeTest", "[ANNLayerTest]") +// { +// arma::mat input(10, 1); +// input.randu(); +// +// arma::mat output1; +// arma::mat output2; +// arma::mat output3; +// arma::mat output4; +// +// const size_t numMergeModules = math::RandInt(2, 10); +// +// MultiplyMerge<> *module1 = new MultiplyMerge<>(true, false); +// for (size_t m = 0; m < numMergeModules; ++m) +// { +// IdentityLayer<> identityLayer; +// identityLayer.Forward(input, identityLayer.OutputParameter()); +// +// module1->Add >(identityLayer); +// } +// +// module1->Forward(input, output1); +// +// MultiplyMerge<> module2 = *module1; +// delete module1; +// +// module2.Forward(input, output2); +// CheckMatrices(output1, output2); +// +// MultiplyMerge<> *module3 = new MultiplyMerge<>(true, false); +// for (size_t m = 0; m < numMergeModules; ++m) +// { +// IdentityLayer<> identityLayer; +// identityLayer.Forward(input, identityLayer.OutputParameter()); +// +// module3->Add >(identityLayer); +// } +// module3->Forward(input, output3); +// +// MultiplyMerge<> module4(std::move(*module3)); +// delete module3; +// +// module4.Forward(input, output4); +// CheckMatrices(output3, output4); +// } + +// /** +// * Simple Atrous Convolution layer test. +// */ +// TEST_CASE("SimpleAtrousConvolutionLayerTest", "[ANNLayerTest]") +// { +// arma::mat output, input, delta; + +// AtrousConvolution<> module1(1, 1, 3, 3, 1, 1, 0, 0, 7, 7, 2, 2); +// // Test the Forward function. +// input = arma::linspace(0, 48, 49); +// module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); +// module1.Parameters()(0) = 1.0; +// module1.Parameters()(8) = 2.0; +// module1.Reset(); +// module1.Forward(input, output); +// // Value calculated using tensorflow.nn.atrous_conv2d() +// REQUIRE(arma::accu(output) == 792.0); + +// // Test the Backward function. +// module1.Backward(input, output, delta); +// REQUIRE(arma::accu(delta) == 2376); + +// AtrousConvolution<> module2(1, 1, 3, 3, 2, 2, 0, 0, 7, 7, 2, 2); +// // Test the forward function. +// input = arma::linspace(0, 48, 49); +// module2.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); +// module2.Parameters()(0) = 1.0; +// module2.Parameters()(3) = 1.0; +// module2.Parameters()(6) = 1.0; +// module2.Reset(); +// module2.Forward(input, output); +// // Value calculated using tensorflow.nn.conv2d() +// REQUIRE(arma::accu(output) == 264.0); + +// // Test the backward function. +// module2.Backward(input, output, delta); +// REQUIRE(arma::accu(delta) == 792.0); +// } + +// /** +// * Atrous Convolution layer numerical gradient test. +// */ +// TEST_CASE("GradientAtrousConvolutionLayerTest", "[ANNLayerTest]") +// { +// // Add function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::linspace(0, 35, 36)), +// target(arma::mat("0")) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(1, 1, 3, 3, 1, 1, 0, 0, 6, 6, 2, 2); +// model->Add >(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// FFN* model; +// arma::mat input, target; +// } function; + +// // TODO: this tolerance seems far higher than necessary. The implementation +// // should be checked. +// REQUIRE(CheckGradient(function) <= 0.2); +// } + +// /** +// * Test the functions to access and modify the parameters of the +// * AtrousConvolution layer. +// */ +// TEST_CASE("AtrousConvolutionLayerParametersTest", "[ANNLayerTest]") +// { +// // Parameter order for the constructor: inSize, outSize, kW, kH, dW, dH, padW, +// // padH, inputWidth, inputHeight, dilationW, dilationH, paddingType ("none"). +// AtrousConvolution<> layer1(1, 2, 3, 4, 5, 6, std::make_tuple(7, 8), +// std::make_tuple(9, 10), 11, 12, 13, 14); +// AtrousConvolution<> layer2(2, 3, 4, 5, 6, 7, std::make_tuple(8, 9), +// std::make_tuple(10, 11), 12, 13, 14, 15); + +// // Make sure we can get the parameters successfully. +// REQUIRE(layer1.InputWidth() == 11); +// REQUIRE(layer1.InputHeight() == 12); +// REQUIRE(layer1.KernelWidth() == 3); +// REQUIRE(layer1.KernelHeight() == 4); +// REQUIRE(layer1.StrideWidth() == 5); +// REQUIRE(layer1.StrideHeight() == 6); +// REQUIRE(layer1.Padding().PadHTop() == 9); +// REQUIRE(layer1.Padding().PadHBottom() == 10); +// REQUIRE(layer1.Padding().PadWLeft() == 7); +// REQUIRE(layer1.Padding().PadWRight() == 8); +// REQUIRE(layer1.DilationWidth() == 13); +// REQUIRE(layer1.DilationHeight() == 14); + +// // Now modify the parameters to match the second layer. +// layer1.InputWidth() = 12; +// layer1.InputHeight() = 13; +// layer1.KernelWidth() = 4; +// layer1.KernelHeight() = 5; +// layer1.StrideWidth() = 6; +// layer1.StrideHeight() = 7; +// layer1.Padding().PadHTop() = 10; +// layer1.Padding().PadHBottom() = 11; +// layer1.Padding().PadWLeft() = 8; +// layer1.Padding().PadWRight() = 9; +// layer1.DilationWidth() = 14; +// layer1.DilationHeight() = 15; + +// // Now ensure all results are the same. +// REQUIRE(layer1.InputWidth() == layer2.InputWidth()); +// REQUIRE(layer1.InputHeight() == layer2.InputHeight()); +// REQUIRE(layer1.KernelWidth() == layer2.KernelWidth()); +// REQUIRE(layer1.KernelHeight() == layer2.KernelHeight()); +// REQUIRE(layer1.StrideWidth() == layer2.StrideWidth()); +// REQUIRE(layer1.StrideHeight() == layer2.StrideHeight()); +// REQUIRE(layer1.Padding().PadHTop() == layer2.Padding().PadHTop()); +// REQUIRE(layer1.Padding().PadHBottom() == +// layer2.Padding().PadHBottom()); +// REQUIRE(layer1.Padding().PadWLeft() == +// layer2.Padding().PadWLeft()); +// REQUIRE(layer1.Padding().PadWRight() == +// layer2.Padding().PadWRight()); +// REQUIRE(layer1.DilationWidth() == layer2.DilationWidth()); +// REQUIRE(layer1.DilationHeight() == layer2.DilationHeight()); +// } + +// /** +// * Test that the padding options are working correctly in Atrous Convolution +// * layer. +// */ +// TEST_CASE("AtrousConvolutionLayerPaddingTest", "[ANNLayerTest]") +// { +// arma::mat output, input, delta; + +// // Check valid padding option. +// AtrousConvolution<> module1(1, 1, 3, 3, 1, 1, +// std::tuple(1, 1), std::tuple(1, 1), 7, 7, +// 2, 2, "valid"); + +// // Test the Forward function. +// input = arma::linspace(0, 48, 49); +// module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); +// module1.Reset(); +// module1.Forward(input, output); + +// REQUIRE(arma::accu(output) == 0); +// REQUIRE(output.n_rows == 9); +// REQUIRE(output.n_cols == 1); + +// // Test the Backward function. +// module1.Backward(input, output, delta); + +// // Check same padding option. +// AtrousConvolution<> module2(1, 1, 3, 3, 1, 1, +// std::tuple(0, 0), std::tuple(0, 0), 7, 7, +// 2, 2, "same"); + +// // Test the forward function. +// input = arma::linspace(0, 48, 49); +// module2.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); +// module2.Reset(); +// module2.Forward(input, output); + +// REQUIRE(arma::accu(output) == 0); +// REQUIRE(output.n_rows == 49); +// REQUIRE(output.n_cols == 1); + +// // Test the backward function. +// module2.Backward(input, output, delta); +// } + +/** + * Tests the GroupNorm layer. + */ +// TEST_CASE("GroupNormTest", "[ANNLayerTest]") +// { +// arma::mat input, output, backwardOutput; +// input = { +// { 2, 0, 1 }, +// { 3, 1, 2 }, +// { 5, 1, 3 }, +// { 7, 2, 4 }, +// { 11, 3, 5 }, +// { 13, 5, 6 }, +// { 17, 8, 7 }, +// { 19, 13, 8 } +// }; +// +// GroupNorm<> model(2, 4); +// model.Reset(); +// +// model.Forward(input, output); +// arma::mat result; +// result = { +// { -1.1717001972, -1.4142135482, -1.3416407811 }, +// { -0.6509445540, 0.0000000000 , -0.4472135937 }, +// { 0.3905667324 , 0.0000000000 , 0.4472135937 }, +// { 1.4320780188 , 1.4142135482 , 1.341640781 }, +// { -1.2649110634, -1.1283296293, -1.3416407811 }, +// { -0.6324555317, -0.5973509802, -0.4472135937 }, +// { 0.6324555317 , 0.1991169934 , 0.4472135937 }, +// { 1.2649110634 , 1.5265636161 , 1.3416407811 } +// }; +// +// CheckMatrices(output, result, 1e-5); +// } + +/** + * GroupNorm layer numerical gradient test. + */ +// TEST_CASE("GradientGroupNormTest", "[ANNLayerTest]") +// { +// // Add function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randn(10, 256)), +// target(arma::zeros(1, 256)) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add >(); +// model->Add >(10, 10); +// model->Add >(1, 10); +// model->Add >(10, 2); +// model->Add >(); +// } +// +// ~GradientFunction() +// { +// delete model; +// } +// +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 256, false); +// model->Gradient(model->Parameters(), 0, gradient, 256); +// return error; +// } +// +// arma::mat& Parameters() { return model->Parameters(); } +// +// FFN* model; +// arma::mat input, target; +// } function; +// +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +/** + * Tests the LayerNorm layer. + * +TEST_CASE("LayerNormTest", "[ANNLayerTest]") +{ + arma::mat input, output; + input = { { 5.1, 3.5 }, + { 4.9, 3.0 }, + { 4.7, 3.2 } }; + + LayerNorm model(input.n_rows); + model.Reset(); + + model.Forward(input, output); + arma::mat result; + result = { { 1.2247, 1.2978 }, + { 0, -1.1355 }, + { -1.2247, -0.1622 } }; + + CheckMatrices(output, result, 1e-1); + result.clear(); + + output = model.Mean(); + result = { 4.9000, 3.2333 }; + + CheckMatrices(output, result, 1e-1); + result.clear(); + + output = model.Variance(); + result = { 0.0267, 0.0422 }; + + CheckMatrices(output, result, 1e-1); +} +*/ + +/** + * LayerNorm layer numerical gradient test. + * +TEST_CASE("GradientLayerNormTest", "[ANNLayerTest]") +{ + // Add function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randn(10, 256)), + target(arma::zeros(1, 256)) + { + model = new FFN(); + model->ResetData(input, target); + model->Add(); + model->Add(10, 10); + model->Add(10); + model->Add(10, 2); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 16, false); + model->Gradient(model->Parameters(), 0, gradient, 16); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + REQUIRE(CheckGradient(function) <= 1e-4); +} +*/ + +/** + * Test that the functions that can access the parameters of the + * Layer Norm layer work. + * +TEST_CASE("LayerNormLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : size, eps. + LayerNorm layer(5, 1e-3); + + // Make sure we can get the parameters successfully. + REQUIRE(layer.InSize() == 5); + REQUIRE(layer.Epsilon() == 1e-3); +} +*/ + +// /** +// * Test if the AddMerge layer is able to forward the +// * Forward/Backward/Gradient calls. +// */ +// TEST_CASE("AddMergeRunTest", "[ANNLayerTest]") +// { +// arma::mat output, input, delta, error; + +// AddMerge<> module(true, true); + +// Linear<>* linear = new Linear<>(10, 10); +// module.Add(linear); + +// linear->Parameters().randu(); +// linear->Reset(); + +// input = arma::zeros(10, 1); +// module.Forward(input, output); + +// double parameterSum = arma::accu(linear->Parameters().submat( +// 100, 0, linear->Parameters().n_elem - 1, 0)); + +// // Test the Backward function. +// module.Backward(input, input, delta); + +// // Clean up before we break, +// delete linear; + +// REQUIRE(parameterSum == Approx(arma::accu(output)).epsilon(1e-5)); +// REQUIRE(arma::accu(delta) == 0); +// } + +/** + * Test if the MultiplyMerge layer is able to forward the + * Forward/Backward/Gradient calls. + * +TEST_CASE("MultiplyMergeRunTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta, error; + + MultiplyMerge module(true, true); + + Linear* linear = new Linear(10, 10); + module.Add(linear); + + linear->Parameters().randu(); + linear->Reset(); + + input = arma::zeros(10, 1); + module.Forward(input, output); + + double parameterSum = arma::accu(linear->Parameters().submat( + 100, 0, linear->Parameters().n_elem - 1, 0)); + + // Test the Backward function. + module.Backward(input, input, delta); + + // Clean up before we break, + delete linear; + + REQUIRE(parameterSum == Approx(arma::accu(output)).epsilon(1e-5)); + REQUIRE(arma::accu(delta) == 0); +} +*/ + +/** + * Simple subview module test. + * +TEST_CASE("SimpleSubviewLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta, outputMat; + Subview moduleRow(1, 10, 19); + + // Test the Forward function for a vector. + input = arma::ones(20, 1); + moduleRow.Forward(input, output); + REQUIRE(output.n_rows == 10); + + Subview moduleMat(4, 3, 6, 0, 2); + + // Test the Forward function for a matrix. + input = arma::ones(20, 8); + moduleMat.Forward(input, outputMat); + REQUIRE(outputMat.n_rows == 12); + REQUIRE(outputMat.n_cols == 2); + + // Test the Backward function. + moduleMat.Backward(input, input, delta); + REQUIRE(accu(delta) == 160); + REQUIRE(delta.n_rows == 20); +} +*/ + +/** + * Subview index test. + * +TEST_CASE("SubviewIndexTest", "[ANNLayerTest]") +{ + arma::mat outputEnd, outputMid, outputStart, input, delta; + input = arma::linspace(1, 20, 20); + + // Slicing from the initial indices. + Subview moduleStart(1, 0, 9); + arma::mat subStart = arma::linspace(1, 10, 10); + + moduleStart.Forward(input, outputStart); + CheckMatrices(outputStart, subStart); + + // Slicing from the mid indices. + Subview moduleMid(1, 6, 15); + arma::mat subMid = arma::linspace(7, 16, 10); + + moduleMid.Forward(input, outputMid); + CheckMatrices(outputMid, subMid); + + // Slicing from the end indices. + Subview moduleEnd(1, 10, 19); + arma::mat subEnd = arma::linspace(11, 20, 10); + + moduleEnd.Forward(input, outputEnd); + CheckMatrices(outputEnd, subEnd); +} +*/ + +/** + * Subview batch test. + * +TEST_CASE("SubviewBatchTest", "[ANNLayerTest]") +{ + arma::mat output, input, outputCol, outputMat, outputDef; + + // All rows selected. + Subview moduleCol(1, 0, 19); + + // Test with inSize 1. + input = arma::ones(20, 8); + moduleCol.Forward(input, outputCol); + CheckMatrices(outputCol, input); + + // Few rows and columns selected. + Subview moduleMat(4, 3, 6, 0, 2); + + // Test with inSize greater than 1. + moduleMat.Forward(input, outputMat); + output = arma::ones(12, 2); + CheckMatrices(outputMat, output); + + // endCol changed to 3 by default. + Subview moduleDef(4, 1, 6, 0, 4); + + // Test with inSize greater than 1 and endCol >= inSize. + moduleDef.Forward(input, outputDef); + output = arma::ones(24, 2); + CheckMatrices(outputDef, output); +} +*/ + +/** + * Test that the functions that can modify and access the parameters of the + * Subview layer work. + * +TEST_CASE("SubviewLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : inSize, beginRow, endRow, beginCol, endCol. + Subview layer1(1, 2, 3, 4, 5); + Subview layer2(1, 3, 4, 5, 6); + + // Make sure we can get the parameters correctly. + REQUIRE(layer1.InSize() == 1); + REQUIRE(layer1.BeginRow() == 2); + REQUIRE(layer1.EndRow() == 3); + REQUIRE(layer1.BeginCol() == 4); + REQUIRE(layer1.EndCol() == 5); + + // Now modify the parameters to match the second layer. + layer1.BeginRow() = 3; + layer1.EndRow() = 4; + layer1.BeginCol() = 5; + layer1.EndCol() = 6; + + // Now ensure all results are the same. + REQUIRE(layer1.InSize() == layer2.InSize()); + REQUIRE(layer1.BeginRow() == layer2.BeginRow()); + REQUIRE(layer1.EndRow() == layer2.EndRow()); + REQUIRE(layer1.BeginCol() == layer2.BeginCol()); + REQUIRE(layer1.EndCol() == layer2.EndCol()); +} +*/ + +/* + * Simple Reparametrization module test. + * +TEST_CASE("SimpleReparametrizationLayerTest", "[ANNLayerTest]") +{ + arma::mat input, output, delta; + Reparametrization module(5); + + // Test the Forward function. As the mean is zero and the standard + // deviation is small, after multiplying the gaussian sample, the + // output should be small enough. + input = join_cols(arma::ones(5, 1) * -15, + arma::zeros(5, 1)); + module.Forward(input, output); + REQUIRE(arma::accu(output) <= 1e-5); + + // Test the Backward function. + arma::mat gy = arma::zeros(5, 1); + module.Backward(input, gy, delta); + REQUIRE(arma::accu(delta) != 0); // klBackward will be added. +} +*/ + +/** + * Reparametrization module stochastic boolean test. + * +TEST_CASE("ReparametrizationLayerStochasticTest", "[ANNLayerTest]") +{ + arma::mat input, outputA, outputB; + Reparametrization module(5, false); + + input = join_cols(arma::ones(5, 1), + arma::zeros(5, 1)); + + // Test if two forward passes generate same output. + module.Forward(input, outputA); + module.Forward(input, outputB); + + CheckMatrices(outputA, outputB); +} +*/ + +/** + * Reparametrization module includeKl boolean test. + * +TEST_CASE("ReparametrizationLayerIncludeKlTest", "[ANNLayerTest]") +{ + arma::mat input, output, gy, delta; + Reparametrization module(5, true, false); + + input = join_cols(arma::ones(5, 1), + arma::zeros(5, 1)); + module.Forward(input, output); + + // As KL divergence is not included, with the above inputs, the delta + // matrix should be all zeros. + gy = arma::zeros(output.n_rows, output.n_cols); + module.Backward(output, gy, delta); + + REQUIRE(arma::accu(delta) == 0); +} +*/ + +/** + * Jacobian Reparametrization module test. + * +TEST_CASE("JacobianReparametrizationLayerTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t inputElementsHalf = math::RandInt(2, 10); + + arma::mat input; + input.set_size(inputElementsHalf * 2, 1); + + Reparametrization module(inputElementsHalf, false, false); + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} +*/ + +/** + * Reparametrization layer numerical gradient test. + * +TEST_CASE("GradientReparametrizationLayerTest", "[ANNLayerTest]") +{ + // Linear function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(10, 1)), + target(arma::mat("0")) + { + model = new FFN(); + model->ResetData(input, target); + model->Add(); + model->Add(10, 6); + model->Add(3, false, true, 1); + model->Add(3, 2); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + // REQUIRE(CheckGradient(function) <= 1e-4); +} +*/ + +/** + * Reparametrization layer beta numerical gradient test. + * +TEST_CASE("GradientReparametrizationLayerBetaTest", "[ANNLayerTest]") +{ + // Linear function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(10, 2)), + target(arma::mat("0 0")) + { + model = new FFN(); + model->ResetData(input, target); + model->Add(); + model->Add(10, 6); + // Use a value of beta not equal to 1. + model->Add(3, false, true, 2); + model->Add(3, 2); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + // REQUIRE(CheckGradient(function) <= 1e-4); +} +*/ + +/** + * Test that the functions that can access the parameters of the + * Reparametrization layer work. + * +TEST_CASE("ReparametrizationLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : latentSize, stochastic, includeKL, beta. + Reparametrization layer(5, false, false, 2); + + // Make sure we can get the parameters successfully. + REQUIRE(layer.OutputSize() == 5); + REQUIRE(layer.Stochastic() == false); + REQUIRE(layer.IncludeKL() == false); + REQUIRE(layer.Beta() == 2); +} +*/ + +/** + * Simple residual module test. + * +TEST_CASE("SimpleResidualLayerTest", "[ANNLayerTest]") +{ + arma::mat outputA, outputB, input, deltaA, deltaB; + + Sequential* sequential = new Sequential(true); + Residual* residual = new Residual(true); + + Linear* linearA = new Linear(10, 10); + linearA->Parameters().randu(); + linearA->Reset(); + Linear* linearB = new Linear(10, 10); + linearB->Parameters().randu(); + linearB->Reset(); + + // Add the same layers (with the same parameters) to both Sequential and + // Residual object. + sequential->Add(linearA); + sequential->Add(linearB); + + residual->Add(linearA); + residual->Add(linearB); + + // Test the Forward function (pass the same input to both). + input = arma::randu(10, 1); + sequential->Forward(input, outputA); + residual->Forward(input, outputB); + + CheckMatrices(outputA, outputB - input); + + // Test the Backward function (pass the same error to both). + sequential->Backward(input, input, deltaA); + residual->Backward(input, input, deltaB); + + CheckMatrices(deltaA, deltaB - input); + + delete sequential; + delete residual; + delete linearA; + delete linearB; +} +*/ + +/** + * Simple Highway module test. + * +TEST_CASE("SimpleHighwayLayerTest", "[ANNLayerTest]") +{ + arma::mat outputA, outputB, input, deltaA, deltaB; + Sequential* sequential = new Sequential(true); + Highway* highway = new Highway(10, true); + highway->Parameters().zeros(); + highway->Reset(); + + Linear* linearA = new Linear(10, 10); + linearA->Parameters().randu(); + linearA->Reset(); + Linear* linearB = new Linear(10, 10); + linearB->Parameters().randu(); + linearB->Reset(); + + // Add the same layers (with the same parameters) to both Sequential and + // Highway object. + highway->Add(linearA); + highway->Add(linearB); + sequential->Add(linearA); + sequential->Add(linearB); + + // Test the Forward function (pass the same input to both). + input = arma::randu(10, 1); + sequential->Forward(input, outputA); + highway->Forward(input, outputB); + + CheckMatrices(outputB, input * 0.5 + outputA * 0.5); + + delete sequential; + delete highway; + delete linearA; + delete linearB; +} +*/ + +/** + * Test that the function that can access the inSize parameter of the + * Highway layer works. + * +TEST_CASE("HighwayLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : inSize, model. + Highway layer(1, true); + + // Make sure we can get the parameter successfully. + REQUIRE(layer.InSize() == 1); +} +*/ + +// /** +// * Sequential layer numerical gradient test. +// */ +// TEST_CASE("GradientHighwayLayerTest", "[ANNLayerTest]") +// { +// // Linear function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(5, 1)), +// target(arma::mat("0")) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add(); +// model->Add(5, 10); + +// highway = new Highway(10); +// highway->Add(10, 10); +// highway->Add(); +// highway->Add(10, 10); +// highway->Add(); + +// model->Add(highway); +// model->Add(10, 2); +// model->Add(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// FFN* model; +// Highway* highway; +// arma::mat input, target; +// } function; + +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +/** + * Sequential layer numerical gradient test. + */ +// TEST_CASE("GradientSequentialLayerTest", "[ANNLayerTest]") +// { +// // Linear function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(10, 1)), +// target(arma::mat("0")) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add(); +// model->Add(10, 10); +// sequential = new Sequential(); +// sequential->Add(10, 10); +// sequential->Add(); +// sequential->Add(10, 5); +// sequential->Add(); + +// model->Add(sequential); +// model->Add(5, 2); +// model->Add(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// FFN* model; +// Sequential* sequential; +// arma::mat input, target; +// } function; + +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +// /** +// * WeightNorm layer numerical gradient test. +// */ +// TEST_CASE("GradientWeightNormLayerTest", "[ANNLayerTest]") +// { +// // Linear function gradient instantiation. +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randu(10, 1)), +// target(arma::mat("0")) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add(10, 10); + +// Linear* linear = new Linear(10, 2); +// weightNorm = new WeightNorm(linear); + +// model->Add(weightNorm); +// model->Add(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1); +// model->Gradient(model->Parameters(), 0, gradient, 1); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// FFN* model; +// WeightNorm* weightNorm; +// arma::mat input, target; +// } function; + +// REQUIRE(CheckGradient(function) <= 1e-4); +// } + +// /** +// * Test if the WeightNorm layer is able to forward the +// * Forward/Backward/Gradient calls. +// */ +// TEST_CASE("WeightNormRunTest", "[ANNLayerTest]") +// { +// arma::mat output, input, delta, error; +// Linear* linear = new Linear(10, 10); + +// WeightNorm module(linear); + +// module.Parameters().randu(); +// module.Reset(); + +// linear->Bias().zeros(); + +// input = arma::zeros(10, 1); +// module.Forward(input, output); + +// // Test the Backward function. +// module.Backward(input, input, delta); + +// REQUIRE(0 == arma::accu(output)); +// REQUIRE(arma::accu(delta) == 0); +// } + +// // General ANN serialization test. +// template +// void ANNLayerSerializationTest(LayerType& layer) +// { +// arma::mat input(5, 100, arma::fill::randu); +// arma::mat output(5, 100, arma::fill::randu); + +// FFN model; +// model.Add>(input.n_rows, 10); +// model.Add(layer); +// model.Add>(); +// model.Add>(10, output.n_rows); +// model.Add>(); + +// ens::StandardSGD opt(0.1, 1, 5, -100, false); +// model.Train(input, output, opt); + +// arma::mat originalOutput; +// model.Predict(input, originalOutput); + +// // Now serialize the model. +// FFN xmlModel, jsonModel, +// binaryModel; +// SerializeObjectAll(model, xmlModel, jsonModel, binaryModel); + +// // Ensure that predictions are the same. +// arma::mat modelOutput, xmlOutput, jsonOutput, binaryOutput; +// model.Predict(input, modelOutput); +// xmlModel.Predict(input, xmlOutput); +// jsonModel.Predict(input, jsonOutput); +// binaryModel.Predict(input, binaryOutput); + +// CheckMatrices(originalOutput, modelOutput, 1e-5); +// CheckMatrices(originalOutput, xmlOutput, 1e-5); +// CheckMatrices(originalOutput, jsonOutput, 1e-5); +// CheckMatrices(originalOutput, binaryOutput, 1e-5); +// } + +// /** +// * Simple serialization test for batch normalization layer. +// */ +// TEST_CASE("BatchNormSerializationTest", "[ANNLayerTest]") +// { +// BatchNorm<> layer(10); +// ANNLayerSerializationTest(layer); +// } + +// /** +// * Simple serialization test for layer normalization layer. +// */ +// TEST_CASE("LayerNormSerializationTest", "[ANNLayerTest]") +// { +// LayerNorm<> layer(10); +// ANNLayerSerializationTest(layer); +// } + +/** + * Test that the padding options in Transposed Convolution layer. + * +TEST_CASE("TransposedConvolutionLayerPaddingTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + + TransposedConvolution module1(1, 1, 3, 3, 1, 1, 0, 0, 4, 4, 6, 6, "VALID"); + // Test the forward function. + // Valid Should give the same result. + input = arma::linspace(0, 15, 16); + module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module1.Reset(); + module1.Forward(input, output); + // Value calculated using tensorflow.nn.conv2d_transpose(). + REQUIRE(arma::accu(output) == 0.0); + + // Test the Backward Function. + module1.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 0.0); + + // Test Valid for non zero padding. + TransposedConvolution module2(1, 1, 3, 3, 2, 2, + std::tuple(0, 0), std::tuple(0, 0), + 2, 2, 5, 5, "VALID"); + // Test the forward function. + input = arma::linspace(0, 3, 4); + module2.Parameters() = arma::mat(25 + 1, 1, arma::fill::zeros); + module2.Parameters()(2) = 8.0; + module2.Parameters()(4) = 6.0; + module2.Parameters()(6) = 4.0; + module2.Parameters()(8) = 2.0; + module2.Reset(); + module2.Forward(input, output); + // Value calculated using torch.nn.functional.conv_transpose2d(). + REQUIRE(arma::accu(output) == 120.0); + + // Test the Backward Function. + module2.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 960.0); + + // Test for same padding type. + TransposedConvolution module3(1, 1, 3, 3, 2, 2, 0, 0, 3, 3, 3, 3, "SAME"); + // Test the forward function. + input = arma::linspace(0, 8, 9); + module3.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module3.Reset(); + module3.Forward(input, output); + REQUIRE(arma::accu(output) == 0); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + + // Test the Backward Function. + module3.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 0.0); + + // Output shape should equal input. + TransposedConvolution module4(1, 1, 3, 3, 1, 1, + std::tuple(2, 2), std::tuple(2, 2), + 5, 5, 5, 5, "SAME"); + // Test the forward function. + input = arma::linspace(0, 24, 25); + module4.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module4.Reset(); + module4.Forward(input, output); + REQUIRE(arma::accu(output) == 0); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + + // Test the Backward Function. + module4.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 0.0); + + TransposedConvolution module5(1, 1, 3, 3, 2, 2, 0, 0, 2, 2, 2, 2, "SAME"); + // Test the forward function. + input = arma::linspace(0, 3, 4); + module5.Parameters() = arma::mat(25 + 1, 1, arma::fill::zeros); + module5.Reset(); + module5.Forward(input, output); + REQUIRE(arma::accu(output) == 0); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + + // Test the Backward Function. + module5.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 0.0); + + TransposedConvolution module6(1, 1, 4, 4, 1, 1, 1, 1, 5, 5, 5, 5, "SAME"); + // Test the forward function. + input = arma::linspace(0, 24, 25); + module6.Parameters() = arma::mat(16 + 1, 1, arma::fill::zeros); + module6.Reset(); + module6.Forward(input, output); + REQUIRE(arma::accu(output) == 0); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + + // Test the Backward Function. + module6.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 0.0); +} +*/ + +/** + * Simple test for Lp Pooling layer. + */ +// TEST_CASE("LpMaxPoolingTestCase", "[ANNLayerTest]") +// { +// // For rectangular input to pooling layers. +// arma::mat input = arma::mat(8, 1); +// arma::mat output; +// input.zeros(); +// input(0) = input(6) = 30; +// input(1) = input(7) = 120; +// input(2) = input(4) = 272; +// input(3) = input(5) = 315; +// // Output-Size should be 1 x 2. +// // Square output. +// LpPooling<> module1(4, 2, 2, 2, 2); +// module1.InputHeight() = 2; +// module1.InputWidth() = 4; +// module1.Forward(input, output); +// // Calculated using torch.nn.LPPool2d(). +// REQUIRE(arma::accu(output) - 706.0 == Approx(0.0).margin(2e-5)); +// REQUIRE(output.n_elem == 2); +// +// // For Square input. +// input = arma::mat(16, 1); +// input.zeros(); +// input(0) = 4; +// input(1) = 3; +// input(3) = 12; +// input(7) = 35; +// input(8) = 6; +// input(11) = 7; +// input(12) = 8; +// input(15) = 24; +// // Output-Size should be 2 x 2. +// // Square output. +// LpPooling<> module3(2, 2, 2, 2, 2); +// module3.InputHeight() = 4; +// module3.InputWidth() = 4; +// module3.Forward(input, output); +// // Calculated using torch.nn.LPPool2d(). +// REQUIRE(arma::accu(output) - 77.0 == Approx(0.0).margin(2e-5)); +// REQUIRE(output.n_elem == 4); +// } + +/** + * Simple test for Mean Pooling layer. + */ +// TEST_CASE("MeanPoolingTestCase", "[ANNLayerTest]") +// { +// // For rectangular input to pooling layers. +// arma::mat input = arma::mat(28, 1); +// input.zeros(); +// input(0) = input(16) = 1; +// input(1) = input(17) = 2; +// input(2) = input(18) = 3; +// input(3) = input(19) = 4; +// input(4) = input(20) = 5; +// input(5) = input(23) = 6; +// input(6) = input(24) = 7; +// input(14) = input(25) = 8; +// input(15) = input(26) = 9; +// +// MeanPooling<> module1(2, 2, 2, 2, false); +// MeanPooling<> module2(2, 2, 2, 2, true); +// module1.InputWidth() = 7; +// module1.InputHeight() = 4; +// module2.InputWidth() = 7; +// module2.InputHeight() = 4; +// +// // Calculated using torch.nn.MeanPool2d(). +// arma::mat result1, result2; +// result1 << 0.7500 << 4.2500 << arma::endr +// << 1.7500 << 4.0000 << arma::endr +// << 2.7500 << 6.0000 << arma::endr +// << 3.5000 << 2.5000 << arma::endr; +// +// result2 << 0.7500 << 4.2500 << arma::endr +// << 1.7500 << 4.0000 << arma::endr +// << 2.7500 << 6.0000 << arma::endr; +// +// arma::mat output1, output2; +// module1.Forward(input, output1); +// module2.Forward(input, output2); +// output1.reshape(4, 2); +// output2.reshape(3, 2); +// CheckMatrices(output1, result1, 1e-1); +// CheckMatrices(output2, result2, 1e-1); +// +// arma::mat delta1, delta2; +// module1.Backward(input, output1, delta1); +// REQUIRE(arma::accu(delta1) == 25.5); +// module2.Backward(input, output2, delta2); +// REQUIRE(arma::accu(delta2) == 19.5); +// } + +/** + * Test that the functions that can modify and access the parameters of the + * Glimpse layer work. + * +TEST_CASE("GlimpseLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : inSize, size, depth, scale, inputWidth, inputHeight. + Glimpse layer1(1, 2, 3, 4, 5, 6); + Glimpse layer2(1, 2, 3, 4, 6, 7); + + // Make sure we can get the parameters successfully. + REQUIRE(layer1.InputHeight() == 6); + REQUIRE(layer1.InputWidth() == 5); + REQUIRE(layer1.Scale() == 4); + REQUIRE(layer1.Depth() == 3); + REQUIRE(layer1.GlimpseSize() == 2); + REQUIRE(layer1.InSize() == 1); + + // Now modify the parameters to match the second layer. + layer1.InputHeight() = 7; + layer1.InputWidth() = 6; + + // Now ensure that all the results are the same. + REQUIRE(layer1.InputHeight() == layer2.InputHeight()); + REQUIRE(layer1.InputWidth() == layer2.InputWidth()); + REQUIRE(layer1.Scale() == layer2.Scale()); + REQUIRE(layer1.Depth() == layer2.Depth()); + REQUIRE(layer1.GlimpseSize() == layer2.GlimpseSize()); + REQUIRE(layer1.InSize() == layer2.InSize()); +} +*/ + +/** + * Test that the function that can access the stdev parameter of the + * Reinforce Normal layer works. + * +TEST_CASE("ReinforceNormalLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter : stdev. + ReinforceNormal layer(4.0); + + // Make sure we can get the parameter successfully. + REQUIRE(layer.StandardDeviation() == 4.0); +} +*/ + +/** + * Simple test for Adaptive pooling for Max Pooling layer. + * +TEST_CASE("AdaptiveMaxPoolingTestCase", "[ANNLayerTest]") +{ + // For rectangular input. + arma::mat input = arma::mat(12, 1); + arma::mat output, delta; + + input.zeros(); + input(0) = 1; + input(1) = 2; + input(2) = 3; + input(3) = input(8) = 7; + input(4) = 4; + input(5) = 5; + input(6) = input(7) = 6; + input(10) = 8; + input(11) = 9; + // Output-Size should be 2 x 2. + // Square output. + AdaptiveMaxPooling module1(2, 2); + module1.InputHeight() = 3; + module1.InputWidth() = 4; + module1.Forward(input, output); + // Calculated using torch.nn.AdaptiveMaxPool2d(). + REQUIRE(arma::accu(output) == 28); + REQUIRE(output.n_elem == 4); + REQUIRE(output.n_cols == 1); + // Test the Backward Function. + module1.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 28.0); + + // For Square input. + input = arma::mat(9, 1); + input.zeros(); + input(0) = 6; + input(1) = 3; + input(2) = 9; + input(3) = 3; + input(6) = 3; + // Output-Size should be 1 x 2. + // Rectangular output. + AdaptiveMaxPooling module2(2, 1); + module2.InputHeight() = 3; + module2.InputWidth() = 3; + module2.Forward(input, output); + // Calculated using torch.nn.AdaptiveMaxPool2d(). + REQUIRE(arma::accu(output) == 15.0); + REQUIRE(output.n_elem == 2); + REQUIRE(output.n_cols == 1); + // Test the Backward Function. + module2.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 15.0); + + // For Square input. + input = arma::mat(16, 1); + input.zeros(); + input(0) = 6; + input(1) = 3; + input(2) = 9; + input(4) = 3; + input(8) = 3; + // Output-Size should be 3 x 3. + // Square output. + AdaptiveMaxPooling module3(std::tuple(3, 3)); + module3.InputHeight() = 4; + module3.InputWidth() = 4; + module3.Forward(input, output); + // Calculated using torch.nn.AdaptiveMaxPool2d(). + REQUIRE(arma::accu(output) == 30.0); + REQUIRE(output.n_elem == 9); + REQUIRE(output.n_cols == 1); + // Test the Backward Function. + module3.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 30.0); + + // For Rectangular input. + input = arma::mat(20, 1); + input.zeros(); + input(0) = 1; + input(1) = 1; + input(3) = 1; + // Output-Size should be 2 x 2. + // Square output. + AdaptiveMaxPooling module4(std::tuple(2, 2)); + module4.InputHeight() = 4; + module4.InputWidth() = 5; + module4.Forward(input, output); + // Calculated using torch.nn.AdaptiveMaxPool2d(). + REQUIRE(arma::accu(output) == 2); + REQUIRE(output.n_elem == 4); + REQUIRE(output.n_cols == 1); + // Test the Backward Function. + module4.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 2.0); +} +*/ + +/** + * Simple test for Adaptive pooling for Mean Pooling layer. + * +TEST_CASE("AdaptiveMeanPoolingTestCase", "[ANNLayerTest]") +{ + // For rectangular input. + arma::mat input = arma::mat(12, 1); + arma::mat output, delta; + + input.zeros(); + input(0) = 1; + input(1) = 2; + input(2) = 3; + input(3) = input(8) = 7; + input(4) = 4; + input(5) = 5; + input(6) = input(7) = 6; + input(10) = 8; + input(11) = 9; + // Output-Size should be 2 x 2. + // Square output. + AdaptiveMeanPooling module1(2, 2); + module1.InputHeight() = 3; + module1.InputWidth() = 4; + module1.Forward(input, output); + // Calculated using torch.nn.AdaptiveAvgPool2d(). + REQUIRE(arma::accu(output) == 19.75); + REQUIRE(output.n_elem == 4); + REQUIRE(output.n_cols == 1); + // Test the Backward Function. + module1.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 19.75); + + // For Square input. + input = arma::mat(9, 1); + input.zeros(); + input(0) = 6; + input(1) = 3; + input(2) = 9; + input(3) = 3; + input(6) = 3; + // Output-Size should be 1 x 2. + // Rectangular output. + AdaptiveMeanPooling module2(1, 2); + module2.InputHeight() = 3; + module2.InputWidth() = 3; + module2.Forward(input, output); + // Calculated using torch.nn.AdaptiveAvgPool2d(). + REQUIRE(arma::accu(output) == 4.5); + REQUIRE(output.n_elem == 2); + REQUIRE(output.n_cols == 1); + // Test the Backward Function. + module2.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 4.50); + + // For Square input. + input = arma::mat(16, 1); + input.zeros(); + input(0) = 6; + input(1) = 3; + input(2) = 9; + input(4) = 3; + input(8) = 3; + // Output-Size should be 3 x 3. + // Square output. + AdaptiveMeanPooling module3(std::tuple(3, 3)); + module3.InputHeight() = 4; + module3.InputWidth() = 4; + module3.Forward(input, output); + // Calculated using torch.nn.AdaptiveAvgPool2d(). + REQUIRE(arma::accu(output) == 10.5); + REQUIRE(output.n_elem == 9); + REQUIRE(output.n_cols == 1); + // Test the Backward Function. + module3.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 10.5); + + // For Rectangular input. + input = arma::mat(24, 1); + input.zeros(); + input(0) = 3; + input(1) = 3; + input(4) = 3; + // Output-Size should be 3 x 3. + // Square output. + AdaptiveMeanPooling module4(std::tuple(3, 3)); + module4.InputHeight() = 4; + module4.InputWidth() = 6; + module4.Forward(input, output); + // Calculated using torch.nn.AdaptiveAvgPool2d(). + REQUIRE(arma::accu(output) == 2.25); + REQUIRE(output.n_elem == 9); + REQUIRE(output.n_cols == 1); + // Test the Backward Function. + module4.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 2.25); +} +*/ + +/* +TEST_CASE("TransposedConvolutionalLayerOptionalParameterTest", "[ANNLayerTest]") +{ + Sequential* decoder = new Sequential(); + + // Check if we can create an object without specifying output. + REQUIRE_NOTHROW(decoder->Add(24, 16, + 5, 5, 1, 1, 0, 0, 10, 10)); + + REQUIRE_NOTHROW(decoder->Add(16, 1, + 15, 15, 1, 1, 1, 1, 14, 14)); + + delete decoder; +} +*/ + +// TEST_CASE("BatchNormWithMinBatchesTest", "[ANNLayerTest]") +// { +// arma::mat input, output, result, runningMean, runningVar, delta; + +// // The input test matrix is of the form 3 x 2 x 4 x 1 where +// // number of images are 3 and number of feature maps are 2. +// input = { { 1, 446, 42 }, +// { 2, 16, 63 }, +// { 3, 13, 63 }, +// { 4, 21, 21 }, +// { 1, 13, 11 }, +// { 32, 45, 42 }, +// { 22, 16, 63 }, +// { 32, 13, 42 } }; +// +// // Output calculated using torch.nn.BatchNorm2d(). +// result = { { -0.4786, 3.2634, -0.1338 }, +// { -0.4702, -0.3525, 0.0427 }, +// { -0.4618, -0.3777, 0.0427 }, +// { -0.4534, -0.3104, -0.3104 }, +// { -1.5429, -0.8486, -0.9643 }, +// { 0.2507, 1.0029, 0.8293 }, +// { -0.3279, -0.675, 2.0443 }, +// { 0.2507 , -0.8486 , 0.8293 } }; + +// // Check correctness of batch normalization. +// BatchNorm<> module1(2, 1e-5, false, 0.1); +// module1.Reset(); +// module1.Forward(input, output); +// CheckMatrices(output, result, 1e-1); + +// // Check backward function. +// module1.Backward(input, output, delta); +// REQUIRE(arma::accu(delta) == Approx(0.0102676).epsilon(1e-5)); + +// // Check values for running mean and running variance. +// // Calculated using torch.nn.BatchNorm2d(). +// runningMean = arma::mat(2, 1); +// runningVar = arma::mat(2, 1); +// runningMean(0) = 5.7917; +// runningMean(1) = 2.76667; +// runningVar(0) = 1543.6545; +// runningVar(1) = 33.488; + +// CheckMatrices(runningMean, module1.TrainingMean(), 1e-3); +// CheckMatrices(runningVar, module1.TrainingVariance(), 1e-2); + +// // Check correctness of layer when running mean and variance +// // are updated using cumulative average. +// BatchNorm<> module2(2); +// module2.Reset(); +// module2.Forward(input, output); +// CheckMatrices(output, result, 1e-1); + +// // Check values for running mean and running variance. +// // Calculated using torch.nn.BatchNorm2d(). +// runningMean(0) = 57.9167; +// runningMean(1) = 27.6667; +// runningVar(0) = 15427.5380; +// runningVar(1) = 325.8787; + +// CheckMatrices(runningMean, module2.TrainingMean(), 1e-2); +// CheckMatrices(runningVar, module2.TrainingVariance(), 1e-2); + +// // Check correctness when model is testing. +// arma::mat deterministicOutput; +// module1.Deterministic() = true; +// module1.Forward(input, deterministicOutput); + +// result.clear(); +// result = { { -0.12195, 11.20426, 0.92158 }, +// { -0.0965, 0.259824, 1.4560 }, +// { -0.071054, 0.183567, 1.45607 }, +// { -0.045601, 0.3870852, 0.38708 }, +// { -0.305288, 1.7683, 1.4227 }, +// { 5.05166, 7.29812, 6.7797 }, +// { 3.323614, 2.2867, 10.4086 }, +// { 5.05166, 1.7683, 6.7797 } }; + +// CheckMatrices(result, deterministicOutput, 1e-1); + +// // Check correctness by updating the running mean and variance again. +// module1.Deterministic() = false; + +// // Clean up. +// output.clear(); +// input.clear(); + +// // The input test matrix is of the form 2 x 2 x 3 x 1 where +// // number of images are 2 and number of feature maps are 2. +// input = { { 12, 443 }, +// { 134, 45 }, +// { 11, 13 }, +// { 14, 55 }, +// { 110, 4 }, +// { 1, 45 } }; +// +// result = { { -0.629337, 2.14791 }, +// { 0.156797, -0.416694 }, +// { -0.63578, -0.622893 }, +// { -0.637481, 0.4440386 }, +// { 1.894857, -0.901267 }, +// { -0.980402, 0.180253 } }; + +// module1.Forward(input, output); +// CheckMatrices(result, output, 1e-3); + +// // Check correctness for the second module as well. +// module2.Forward(input, output); +// CheckMatrices(result, output, 1e-3); + +// // Calculated using torch.nn.BatchNorm2d(). +// runningMean(0) = 16.1792; +// runningMean(1) = 6.30667; +// runningVar(0) = 4276.5849; +// runningVar(1) = 202.595; + +// CheckMatrices(runningMean, module1.TrainingMean(), 1e-3); +// CheckMatrices(runningVar, module1.TrainingVariance(), 1e-1); + +// // Check correctness of running mean and variance when their +// // values are updated using cumulative average. +// runningMean(0) = 83.79166; +// runningMean(1) = 32.9166; +// runningVar(0) = 22164.1035; +// runningVar(1) = 1025.2227; + +// CheckMatrices(runningMean, module2.TrainingMean(), 1e-3); +// CheckMatrices(runningVar, module2.TrainingVariance(), 1e-3); + +// // Check backward function. +// module1.Backward(input, output, delta); + +// deterministicOutput.clear(); +// module1.Deterministic() = true; +// module1.Forward(input, deterministicOutput); + +// result.clear(); +// result = { { -0.06388436, 6.524754114 }, +// { 1.799655281, 0.44047968 }, +// { -0.07913291, -0.04784981 }, +// { 0.5405045, 3.4210097 }, +// { 7.2851023, -0.1620577 }, +// { -0.37282639, 2.7184474 } }; + +// // Calculated using torch.nn.BatchNorm2d(). +// CheckMatrices(result, deterministicOutput, 1e-1); +// } + +// /** +// * Batch Normalization layer numerical gradient test. +// */ +// TEST_CASE("GradientBatchNormWithMiniBatchesTest", "[ANNLayerTest]") +// { +// // Add function gradient instantiation. +// // To make this test robust, check it ten times. +// bool pass = false; +// for (size_t trial = 0; trial < 10; trial++) +// { +// struct GradientFunction +// { +// GradientFunction() : +// input(arma::randn(16, 1024)), +// target(arma::zeros(1, 1024)) +// { +// model = new FFN(); +// model->ResetData(input, target); +// model->Add>(); +// model->Add>(1, 2, 3, 3, 1, 1, 0, 0, 4, 4); +// model->Add>(2); +// model->Add>(2 * 2 * 2, 2); +// model->Add>(); +// } + +// ~GradientFunction() +// { +// delete model; +// } + +// double Gradient(arma::mat& gradient) const +// { +// double error = model->Evaluate(model->Parameters(), 0, 1024, false); +// model->Gradient(model->Parameters(), 0, gradient, 1024); +// return error; +// } + +// arma::mat& Parameters() { return model->Parameters(); } + +// FFN* model; +// arma::mat input, target; +// } function; + +// double gradient = CheckGradient(function); +// if (gradient < 1e-1) +// { +// pass = true; +// break; +// } +// } + +// REQUIRE(pass); +// } + +// TEST_CASE("BatchNormDeterministicTest", "[ANNLayerTest]") +// { +// FFN<> module; +// module.Add>(2, 1e-5, false); +// module.Add>(); + +// arma::mat input(4, 3), output; +// module.ResetParameters(); + +// // The model should switch to Deterministic mode for predicting. +// module.Predict(input, output); +// REQUIRE(boost::get*>(module.Model()[0])->Deterministic() == true); + +// output.ones(); +// module.Train(input, output); +// // The model should switch to training mode for predicting. +// REQUIRE(boost::get*>(module.Model()[0])->Deterministic() == 0); +// } + +// /** +// * Linear module weight initialization test. +// */ +// TEST_CASE("LinearLayerWeightInitializationTest", "[ANNLayerTest]") +// { +// size_t inSize = 10, outSize = 4; +// Linear<> linear = Linear<>(inSize, outSize); +// linear.Reset(); +// RandomInitialization().Initialize(linear.Weight()); +// linear.Bias().ones(); + +// REQUIRE(std::equal(linear.Weight().begin(), +// linear.Weight().end(), linear.Parameters().begin())); + +// REQUIRE(std::equal(linear.Bias().begin(), +// linear.Bias().end(), linear.Parameters().begin() + inSize * outSize)); + +// REQUIRE(linear.Weight().n_rows == outSize); +// REQUIRE(linear.Weight().n_cols == inSize); +// REQUIRE(linear.Bias().n_rows == outSize); +// REQUIRE(linear.Bias().n_cols == 1); +// REQUIRE(linear.Parameters().n_rows == inSize * outSize + outSize); +// } + +// /** +// * Atrous Convolution module weight initialization test. +// */ +// TEST_CASE("AtrousConvolutionLayerWeightInitializationTest", "[ANNLayerTest]") +// { +// size_t inSize = 2, outSize = 3; +// size_t kernelWidth = 4, kernelHeight = 5; +// AtrousConvolution<> module = AtrousConvolution<>(inSize, outSize, +// kernelWidth, kernelHeight, 6, 7, std::make_tuple(8, 9), +// std::make_tuple(10, 11), 12, 13, 14, 15); +// module.Reset(); +// RandomInitialization().Initialize(module.Weight()); +// module.Bias().ones(); + +// REQUIRE(std::equal(module.Weight().begin(), +// module.Weight().end(), module.Parameters().begin())); + +// REQUIRE(std::equal(module.Bias().begin(), +// module.Bias().end(), module.Parameters().end() - outSize)); + +// REQUIRE(module.Weight().n_rows == kernelWidth); +// REQUIRE(module.Weight().n_cols == kernelHeight); +// REQUIRE(module.Weight().n_slices == inSize * outSize); +// REQUIRE(module.Bias().n_rows == outSize); +// REQUIRE(module.Bias().n_cols == 1); +// REQUIRE(module.Parameters().n_rows +// == (outSize * inSize * kernelWidth * kernelHeight) + outSize); +// } + +/** + * Transposed Convolution module weight initialization test. + * +TEST_CASE("TransposedConvolutionWeightInitializationTest", "[ANNLayerTest]") +{ + size_t inSize = 3, outSize = 3; + size_t kernelWidth = 4, kernelHeight = 4; + TransposedConvolution module = TransposedConvolution(inSize, outSize, + kernelWidth, kernelHeight, 1, 1, 1, 1, 5, 5, 6, 6); + module.Reset(); + RandomInitialization().Initialize(module.Weight()); + module.Bias().ones(); + + REQUIRE(std::equal(module.Weight().begin(), + module.Weight().end(), module.Parameters().begin())); + + REQUIRE(std::equal(module.Bias().begin(), + module.Bias().end(), module.Parameters().end() - outSize)); + + REQUIRE(module.Weight().n_rows == kernelWidth); + REQUIRE(module.Weight().n_cols == kernelHeight); + REQUIRE(module.Weight().n_slices == inSize * outSize); + REQUIRE(module.Bias().n_rows == outSize); + REQUIRE(module.Bias().n_cols == 1); + REQUIRE(module.Parameters().n_rows + == (outSize * inSize * kernelWidth * kernelHeight) + outSize); +} +*/ + +/** + * Simple Test for ChannelShuffle layer. + */ +// TEST_CASE("ChannelShuffleLayerTest", "[ANNLayerTest]") +// { +// arma::mat input1, output1, outputExpected1, outputBackward1; +// ChannelShuffle<> module1(2, 2, 6, 2); +// +// input1 << 1 << 13 << arma::endr +// << 2 << 14 << arma::endr +// << 3 << 15 << arma::endr +// << 4 << 16 << arma::endr +// << 5 << 17 << arma::endr +// << 6 << 18 << arma::endr +// << 7 << 19 << arma::endr +// << 8 << 20 << arma::endr +// << 9 << 21 << arma::endr +// << 10 << 22 << arma::endr +// << 11 << 23 << arma::endr +// << 12 << 24 << arma::endr; +// input1.reshape(24, 1); +// // Value calculated using torch.nn.ChannelShuffle(). +// outputExpected1 << 1 << 17 << arma::endr +// << 2 << 18 << arma::endr +// << 3 << 19 << arma::endr +// << 4 << 20 << arma::endr +// << 13 << 9 << arma::endr +// << 14 << 10 << arma::endr +// << 15 << 11 << arma::endr +// << 16 << 12 << arma::endr +// << 5 << 21 << arma::endr +// << 6 << 22 << arma::endr +// << 7 << 23 << arma::endr +// << 8 << 24 << arma::endr; +// outputExpected1.reshape(24, 1); +// // Check the Forward pass of the layer. +// module1.Forward(input1, output1); +// CheckMatrices(output1, outputExpected1); +// +// // Check the Backward pass of the layer. +// module1.Backward(output1, output1, outputBackward1); +// CheckMatrices(input1, outputBackward1); +// +// } + +/** + * Simple Test for PixelShuffle layer. + */ +// TEST_CASE("PixelShuffleLayerTest", "[ANNLayerTest]") +// { +// arma::mat input1, output1, gy1, g1, outputExpected1, gExpected1; +// arma::mat input2, output2, gy2, g2, outputExpected2, gExpected2; +// PixelShuffle<> module1(2, 2, 2, 4); +// PixelShuffle<> module2(2, 2, 2, 4); +// +// // Input is a single image, of size (2,2) and having 4 channels. +// input1 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 +// << 0 << 0 << arma::endr; +// gy1 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8 +// << 12 << 16 << arma::endr; +// +// // Calculated using torch.nn.PixelShuffle(). +// outputExpected1 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0 +// << 0 << 0 << 0 << 0 << arma::endr; +// gExpected1 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12 +// << 6 << 14 << 8 << 16 << arma::endr; +// +// input1 = input1.t(); +// outputExpected1 = outputExpected1.t(); +// gy1 = gy1.t(); +// gExpected1 = gExpected1.t(); +// +// // Check the Forward pass of the layer. +// module1.Forward(input1, output1); +// CheckMatrices(output1, outputExpected1); +// +// // Check the Backward pass of the layer. +// module1.Backward(input1, gy1, g1); +// CheckMatrices(g1, gExpected1); +// +// // Input is a batch of 2 images, each of size (2,2) and having 4 channels. +// input2 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 +// << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0 +// << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr; +// gy2 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8 +// << 12 << 16 << arma::endr << 17 << 21 << 25 << 29 << 18 << 22 << 26 << 30 +// << 19 << 23 << 27 << 31 << 20 << 24 << 28 << 32 << arma::endr; +// +// // Calculated using torch.nn.PixelShuffle(). +// outputExpected2 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0 +// << 0 << 0 << 0 << 0 << arma::endr << 5 << 0 << 7 << 0 << 0 << 0 << 0 << 0 +// << 6 << 0 << 8 << 0 << 0 << 0 << 0 << 0 << arma::endr; +// gExpected2 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12 +// << 6 << 14 << 8 << 16 << arma::endr << 17 << 25 << 19 << 27 << 21 << 29 +// << 23 << 31 << 18 << 26 << 20 << 28 << 22 << 30 << 24 << 32 << arma::endr; +// +// input2 = input2.t(); +// outputExpected2 = outputExpected2.t(); +// gy2 = gy2.t(); +// gExpected2 = gExpected2.t(); +// +// // Check the Forward pass of the layer. +// module2.Forward(input2, output2); +// CheckMatrices(output2, outputExpected2); +// +// // Check the Backward pass of the layer. +// module2.Backward(input2, gy2, g2); +// CheckMatrices(g2, gExpected2); +// } + +/** + * Test that the function that can access the parameters of the + * PixelShuffle layer works. + */ +// TEST_CASE("PixelShuffleLayerParametersTest", "[ANNLayerTest]") +// { +// // Create the layer using the empty constructor. +// PixelShuffle<> layer; +// +// // Set the different input parameters of the layer. +// layer.UpscaleFactor() = 2; +// layer.InputHeight() = 2; +// layer.InputWidth() = 2; +// layer.InputChannels() = 4; +// +// // Make sure we can get the parameters successfully. +// REQUIRE(layer.UpscaleFactor() == 2); +// REQUIRE(layer.InputHeight() == 2); +// REQUIRE(layer.InputWidth() == 2); +// REQUIRE(layer.InputChannels() == 4); +// +// arma::mat input, output; +// // Input is a batch of 2 images, each of size (2,2) and having 4 channels. +// input << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 +// << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0 +// << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr; +// input = input.t(); +// layer.Forward(input, output); +// +// // Check whether output parameters are returned correctly. +// REQUIRE(layer.OutputHeight() == 4); +// REQUIRE(layer.OutputWidth() == 4); +// REQUIRE(layer.OutputChannels() == 1); +// } + +// /** +// * Simple Test for SpatialDropout layer. +// */ +// TEST_CASE("SpatialDropoutLayerTest", "[ANNLayerTest]") +// { +// arma::mat input, output, gy, g, temp; +// arma::mat outputsExpected = arma::zeros(8, 12); +// arma::mat gsExpected = arma::zeros(8, 12); + +// // Set the seed to a random value. +// arma::arma_rng::set_seed_random(); +// SpatialDropout<> module(3, 0.2); + +// // Input is a batch of 2 images, each of size (2,2) and having 4 channels. +// input = { 0.4963, 0.0885, 0.7682, 0.1320, 0.3074, 0.4901, 0.6341, 0.8964, +// 0.4556, 0.3489, 0.6323, 0.4017 }; +// +// gy = { 1, 3, 2, 4, 5, 7, 6, 8, 9, 11, 10, 12 }; +// +// // Following values have been calculated using torch.nn.Dropout2d(p=0.2). +// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; +// outputsExpected.row(0) = temp; +// temp = { 0, 0, 0, 0, 0.3842, 0.6126, 0.7926, 1.1205, 0.5695, 0.4361, 0.7904, +// 0.5021 }; +// outputsExpected.row(1) = temp; +// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0, 0, 0, 0, 0.5695, 0.4361, +// 0.7904, 0.5021 }; +// outputsExpected.row(2) = temp; +// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0.3842, 0.6126, 0.7926, 1.1205, 0, +// 0, 0, 0 }; +// outputsExpected.row(3) = temp; +// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 0.5695, 0.4361, 0.7904, 0.5021 }; +// outputsExpected.row(4) = temp; +// temp = { 0, 0, 0, 0, 0.3842, 0.6126, 0.7926, 1.1205, 0, 0, 0, 0 }; +// outputsExpected.row(5) = temp; +// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0, 0, 0, 0, 0, 0, 0, 0 }; +// outputsExpected.row(6) = temp; +// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0.3842, 0.6126, 0.7926, 1.1205, +// 0.5695, 0.4361, 0.7904, 0.5021 }; +// outputsExpected.row(7) = temp; +// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; +// gsExpected.row(0) = temp; +// temp = { 0, 0, 0, 0, 6.2500, 8.7500, 7.5000, 10.0000, 11.2500, 13.7500, +// 12.5000, 15.0000 }; +// gsExpected.row(1) = temp; +// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 0, 0, 0, 0, 11.2500, 13.7500, +// 12.5000, 15.0000 }; +// gsExpected.row(2) = temp; +// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 6.2500, 8.7500, 7.5000, 10.0000, 0, +// 0, 0, 0 }; +// gsExpected.row(3) = temp; +// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 11.2500, 13.7500, 12.5000, 15.0000 }; +// gsExpected.row(4) = temp; +// temp = { 0, 0, 0, 0, 6.2500, 8.7500, 7.5000, 10.0000, 0, 0, 0, 0 }; +// gsExpected.row(5) = temp; +// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 0, 0, 0, 0, 0, 0, 0, 0 }; +// gsExpected.row(6) = temp; +// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 6.2500, 8.7500, 7.5000, 10.0000, +// 11.2500, 13.7500, 12.5000, 15.0000 }; +// gsExpected.row(7) = temp; + +// input = input.t(); +// gy = gy.t(); +// outputsExpected = outputsExpected.t(); +// gsExpected = gsExpected.t(); + +// // Compute the Forward and Backward passes and store the results. +// module.Forward(input, output); +// module.Backward(input, gy, g); + +// // Check through all possible cases, to find a match and then compare results. +// for (size_t i = 0; i < outputsExpected.n_cols; ++i) +// { +// if (arma::approx_equal(outputsExpected.col(i), output, "absdiff", 1e-1)) +// { +// // Check the correctness of the Forward pass of the layer. +// CheckMatrices(output, outputsExpected.col(i), 1e-1); +// // Check the correctness of the Backward pass of the layer. +// CheckMatrices(g, gsExpected.col(i), 1e-1); +// } +// } + +// // Check if the output is same as input when using deterministic mode. +// module.Deterministic() = true; +// output.clear(); +// module.Forward(input, output); +// CheckMatrices(output, input, 1e-1); +// } + +// /** +// * Test that the function that can access the parameters of the +// * SpatialDropout layer works. +// */ +// TEST_CASE("SpatialDropoutLayerParametersTest", "[ANNLayerTest]") +// { +// // Create the layer using the empty constructor. +// SpatialDropout<> layer; + +// // Set the input parameters. +// layer.Size() = 3; +// layer.Ratio(0.2); + +// // Check whether the input parameters have been set correctly. +// REQUIRE(layer.Size() == 3); +// REQUIRE(layer.Ratio() == 0.2); +// } + +/** + * Simple Positional Encoding layer test. + * +TEST_CASE("SimplePositionalEncodingTest", "[ANNLayerTest]") +{ + const size_t seqLength = 5; + const size_t embedDim = 4; + const size_t batchSize = 2; + + arma::mat input = arma::randu(embedDim * seqLength, batchSize); + arma::mat gy = 0.01 * arma::randu(embedDim * seqLength, batchSize); + arma::mat output, g; + + PositionalEncoding module(embedDim, seqLength); + + // Check Forward function. + module.Forward(input, output); + arma::mat pe = output - input; + CheckMatrices(arma::mean(pe, 1), module.Encoding()); + + // Check Backward function. + module.Backward(input, gy, g); + REQUIRE(std::equal(gy.begin(), gy.end(), g.begin())); +} +*/ + +/** + * Jacobian test for Positional Encoding layer. + * +TEST_CASE("JacobianPositionalEncodingTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t embedDim = 4; + const size_t seqLength = math::RandInt(5, 10); + arma::mat input; + input.set_size(embedDim * seqLength, 1); + + PositionalEncoding module(embedDim, seqLength); + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} +*/ + +/** + * Simple Multihead Attention test. + * +TEST_CASE("SimpleMultiheadAttentionTest", "[ANNLayerTest]") +{ + size_t tLen = 5; + size_t sLen = tLen; + size_t embedDim = 4; + size_t numHeads = 2; + size_t bsz = 3; + + arma::mat query = 0.1 * arma::randu(embedDim * tLen, bsz); + arma::mat output; + + arma::mat attnMask = arma::zeros(tLen, sLen); + for (size_t i = 0; i < tLen; ++i) + { + for (size_t j = 0; j < sLen; ++j) + { + if (i < j) + attnMask(i, j) = std::numeric_limits::lowest(); + } + } + + arma::mat keyPaddingMask = arma::zeros(1, sLen); + keyPaddingMask(sLen - 1) = std::numeric_limits::lowest(); + + MultiheadAttention module(tLen, sLen, embedDim, numHeads); + module.AttentionMask() = attnMask; + module.KeyPaddingMask() = keyPaddingMask; + module.Reset(); + module.Parameters().randu(); + + // Forward test. + arma::mat input = arma::join_cols(arma::join_cols(query, query), query); + + module.Forward(input, output); + REQUIRE(output.n_rows == embedDim * tLen); + REQUIRE(output.n_cols == bsz); + + // Backward test. + arma::mat gy = 0.01 * arma::randu(embedDim * tLen, bsz); + arma::mat g; + module.Backward(input, gy, g); + REQUIRE(g.n_rows == input.n_rows); + REQUIRE(g.n_cols == input.n_cols); + + // Gradient test. + arma::mat error = 0.05 * arma::randu(embedDim * tLen, bsz); + arma::mat gradient; + module.Gradient(input, error, gradient); + REQUIRE(gradient.n_rows == module.Parameters().n_rows); + REQUIRE(gradient.n_cols == module.Parameters().n_cols); +} +*/ + +/** + * Jacobian MultiheadAttention module test. + * +TEST_CASE("JacobianMultiheadAttentionTest", "[ANNLayerTest]") +{ + // Check when query = key = value. + for (size_t i = 0; i < 5; ++i) + { + const size_t tgtSeqLen = 2; + const size_t embedDim = 4; + const size_t nHeads = 2; + const size_t batchSize = 1; + + arma::mat query = arma::randu(embedDim * tgtSeqLen, batchSize); + arma::mat input = arma::join_cols(arma::join_cols(query, query), query); + + MultiheadAttention module(tgtSeqLen, tgtSeqLen, embedDim, nHeads); + module.Parameters().randu(); + + double error = CustomJacobianTest(module, input); + REQUIRE(error <= 1e-5); + } + + // Check when key = value. + for (size_t i = 0; i < 5; ++i) + { + const size_t tgtSeqLen = 2; + const size_t srcSeqLen = math::RandInt(2, 5); + const size_t embedDim = 4; + const size_t nHeads = 2; + const size_t batchSize = 1; + + arma::mat query = arma::randu(embedDim * tgtSeqLen, batchSize); + arma::mat key = 0.091 * arma::randu(embedDim * srcSeqLen, batchSize); + arma::mat input = arma::join_cols(arma::join_cols(query, key), key); + + MultiheadAttention module(tgtSeqLen, srcSeqLen, embedDim, nHeads); + module.Parameters().randu(); + + double error = CustomJacobianTest(module, input); + REQUIRE(error <= 1e-5); + } + + // Check when query, key and value are not same. + for (size_t i = 0; i < 5; ++i) + { + const size_t tgtSeqLen = 2; + const size_t srcSeqLen = math::RandInt(2, 5); + const size_t embedDim = 4; + const size_t nHeads = 2; + const size_t batchSize = 1; + + arma::mat query = arma::randu(embedDim * tgtSeqLen, batchSize); + arma::mat key = 0.091 * arma::randu(embedDim * srcSeqLen, batchSize); + arma::mat value = 0.045 * arma::randu(embedDim * srcSeqLen, batchSize); + arma::mat input = arma::join_cols(arma::join_cols(query, key), value); + + MultiheadAttention module(tgtSeqLen, srcSeqLen, embedDim, nHeads); + module.Parameters().randu(); + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} +*/ + +/** + * Numerical gradient test for MultiheadAttention layer. + * +TEST_CASE("GradientMultiheadAttentionTest", "[ANNLayerTest]") +{ + struct GradientFunction + { + GradientFunction() : + tgtSeqLen(2), + srcSeqLen(2), + embedDim(4), + nHeads(2), + vocabSize(5), + batchSize(2) + { + input = arma::randu(embedDim * (tgtSeqLen + 2 * srcSeqLen), batchSize); + target = arma::zeros(vocabSize, batchSize); + for (size_t i = 0; i < target.n_elem; ++i) + { + const size_t label = mlpack::math::RandInt(1, vocabSize); + target(i) = label; + } + + attnMask = arma::zeros(tgtSeqLen, srcSeqLen); + for (size_t i = 0; i < tgtSeqLen; ++i) + { + for (size_t j = 0; j < srcSeqLen; ++j) + { + if (i < j) + attnMask(i, j) = std::numeric_limits::lowest(); + } + } + + keyPaddingMask = arma::zeros(1, srcSeqLen); + keyPaddingMask(srcSeqLen - 1) = std::numeric_limits::lowest(); + + model = new FFN(); + model->ResetData(input, target); + // attnModule = new MultiheadAttention(tgtSeqLen, srcSeqLen, embedDim, + // nHeads); + // attnModule->AttentionMask() = attnMask; + // attnModule->KeyPaddingMask() = keyPaddingMask; + // model->Add(attnModule); + model->Add(tgtSeqLen, srcSeqLen, embedDim, nHeads, + attnMask, keyPaddingMask); + model->Add(embedDim * tgtSeqLen, vocabSize); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, batchSize); + model->Gradient(model->Parameters(), 0, gradient, batchSize); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + // MultiheadAttention* attnModule; + + arma::mat input, target, attnMask, keyPaddingMask; + const size_t tgtSeqLen; + const size_t srcSeqLen; + const size_t embedDim; + const size_t nHeads; + const size_t vocabSize; + const size_t batchSize; + } function; + + REQUIRE(CheckGradient(function) <= 3e-06); +} +*/ + +/** + * Simple tests for instance normalization layer. + * +TEST_CASE("InstanceNormLayerTest", "[ANNLayerTest]") +{ + arma::mat input, result, output, delta, deltaExpected; + arma::mat runningMean, runningVar; + + // Represents 2 images, each having 3 channels, and shape (3,2). + input << 1 << 19 << arma::endr + << 2 << 20 << arma::endr + << 3 << 21 << arma::endr + << 4 << 22 << arma::endr + << 5 << 23 << arma::endr + << 6 << 24 << arma::endr + << 7 << 25 << arma::endr + << 8 << 26 << arma::endr + << 9 << 27 << arma::endr + << 10 << 28 << arma::endr + << 11 << 29 << arma::endr + << 12 << 30 << arma::endr + << 13 << 31 << arma::endr + << 14 << 32 << arma::endr + << 15 << 33 << arma::endr + << 16 << 34 << arma::endr + << 17 << 35 << arma::endr + << 18 << 36 << arma::endr; + + // Output calculated using torch.nn.InstanceNorm2d(). + result << -1.4638 << -1.4638 << arma::endr + << -0.8783 << -0.8783 << arma::endr + << -0.2928 << -0.2928 << arma::endr + << 0.2928 << 0.2928 << arma::endr + << 0.8783 << 0.8783 << arma::endr + << 1.4638 << 1.4638 << arma::endr + << -1.4638 << -1.4638 << arma::endr + << -0.8783 << -0.8783 << arma::endr + << -0.2928 << -0.2928 << arma::endr + << 0.2928 << 0.2928 << arma::endr + << 0.8783 << 0.8783 << arma::endr + << 1.4638 << 1.4638 << arma::endr + << -1.4638 << -1.4638 << arma::endr + << -0.8783 << -0.8783 << arma::endr + << -0.2928 << -0.2928 << arma::endr + << 0.2928 << 0.2928 << arma::endr + << 0.8783 << 0.8783 << arma::endr + << 1.4638 << 1.4638 << arma::endr; + + // Calculated using torch.nn.InstanceNorm2d(). + deltaExpected << 1.8367 << 1.8367 << arma::endr + << 0.3967 << 0.3967 << arma::endr + << 0.0147 << 0.0147 << arma::endr + <<-0.0147 << -0.0147 << arma::endr + <<-0.3967 << -0.3967 << arma::endr + <<-1.8367 << -1.8367 << arma::endr + << 1.8367 << 1.8367 << arma::endr + << 0.3967 << 0.3967 << arma::endr + << 0.0147 << 0.0147 << arma::endr + <<-0.0147 << -0.0147 << arma::endr + <<-0.3967 << -0.3967 << arma::endr + <<-1.8367 << -1.8367 << arma::endr + << 1.8367 << 1.8367 << arma::endr + << 0.3967 << 0.3967 << arma::endr + << 0.0147 << 0.0147 << arma::endr + <<-0.0147 << -0.0147 << arma::endr + <<-0.3967 << -0.3967 << arma::endr + <<-1.8367 << -1.8367 << arma::endr; + + // Check Forward and Backward pass in non-deterministic mode. + InstanceNorm<> module(3, input.n_cols, 1e-5, false, 0.1); + output.zeros(arma::size(input)); + module.Forward(input, output); + CheckMatrices(output, result, 1e-1); + + module.Backward(input, output, delta); + CheckMatrices(delta, deltaExpected, 1e-1); + + runningMean = arma::mat(3, 1); + runningVar = arma::mat(3, 1); + runningMean(0) = 1.2500; + runningMean(1) = 1.8500; + runningMean(2) = 2.4500; + runningVar(0) = 1.2500; + runningVar(1) = 1.2500; + runningVar(2) = 1.2500; + + CheckMatrices(runningMean, module.TrainingMean(), 1e-1); + CheckMatrices(runningVar, module.TrainingVariance(), 1e-1); + + // Check Forward pass in deterministic mode. + InstanceNorm<> module1(3, input.n_cols, 1e-5, false, 0.1); + module1.Deterministic() = true; + output.zeros(arma::size(input)); + module1.Forward(input, output); + + // Calculated using torch.nn.InstanceNorm2d(). + result << 1.0000 << 18.9999 << arma::endr + << 2.0000 << 19.9999 << arma::endr + << 3.0000 << 20.9999 << arma::endr + << 4.0000 << 21.9999 << arma::endr + << 5.0000 << 22.9999 << arma::endr + << 6.0000 << 23.9999 << arma::endr + << 7.0000 << 24.9999 << arma::endr + << 8.0000 << 25.9999 << arma::endr + << 9.0000 << 26.9999 << arma::endr + << 10.0000 << 27.9999 << arma::endr + << 10.9999 << 28.9999 << arma::endr + << 11.9999 << 29.9999 << arma::endr + << 12.9999 << 30.9998 << arma::endr + << 13.9999 << 31.9998 << arma::endr + << 14.9999 << 32.9998 << arma::endr + << 15.9999 << 33.9998 << arma::endr + << 16.9999 << 34.9998 << arma::endr + << 17.9999 << 35.9998 << arma::endr; + + CheckMatrices(output, result, 1e-1); +} +*/ + +/** + * Test that the functions that can access the parameters of the + * Instance Norm layer work. + * +TEST_CASE("InstanceNormLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : size, eps. + InstanceNorm<> layer(7, 0, 1e-3); + + // Make sure we can get the parameters successfully. + REQUIRE(layer.InputSize() == 7); + REQUIRE(layer.Epsilon() == 1e-3); + + arma::mat runningMean(7, 1, arma::fill::randn); + arma::mat runningVariance(7, 1, arma::fill::randn); + + layer.TrainingVariance() = runningVariance; + layer.TrainingMean() = runningMean; + CheckMatrices(layer.TrainingVariance(), runningVariance); + CheckMatrices(layer.TrainingMean(), runningMean); +} +*/ + +/** + * Instance Norm layer numerical gradient test. + * +TEST_CASE("GradientInstanceNormLayerTest", "[ANNLayerTest]") +{ + // Add function gradient instantiation. + // To make this test robust, check it ten times. + bool pass = false; + for (size_t trial = 0; trial < 10; trial++) + { + struct GradientFunction + { + GradientFunction() + { + input = arma::randn(16, 1024); + arma::mat target; + target.ones(1, 1024); + + model = new FFN(); + model->ResetData(input, target); + model->Add >(); + model->Add >(1, 2, 3, 3, 1, 1, 0, 0, 4, 4); + model->Add > (2, 1024); + model->Add >(2 * 2 * 2, 2); + model->Add >(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1024, false); + model->Gradient(model->Parameters(), 0, gradient, 1024); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + double gradient = CheckGradient(function); + if (gradient < 1e-1) + { + pass = true; + break; + } + } + + REQUIRE(pass); +} +*/ diff --git a/src/mlpack/tests/ann/not_adapted/feedforward_network_test.cpp b/src/mlpack/tests/ann/not_adapted/feedforward_network_test.cpp new file mode 100644 index 0000000000..c63f1d33e6 --- /dev/null +++ b/src/mlpack/tests/ann/not_adapted/feedforward_network_test.cpp @@ -0,0 +1,245 @@ +/** + * @file tests/feedforward_network_test.cpp + * @author Marcus Edel + * @author Palash Ahuja + * + * Tests the feed forward network. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include +#include + +#include + +#include "../catch.hpp" +#include "../serialization.hpp" + +using namespace mlpack; +using namespace mlpack::ann; + +/** + * Check whether copying and moving network with Reparametrization is working or not. + * +TEST_CASE("CheckCopyMovingReparametrizationNetworkTest", + "[FeedForwardNetworkTest]") +{ + // Load the dataset. + arma::mat trainData; + data::Load("thyroid_train.csv", trainData, true); + + arma::mat trainLabels = trainData.row(trainData.n_rows - 1); + trainData.shed_row(trainData.n_rows - 1); + + // Construct a feed forward network with trainData.n_rows input nodes, + // followed by a linear layer and then a reparametrization layer. + FFN *model = new FFN; + model->Add(8); + model->Add(false, true, 1); + model->Add(); + + FFN *model1 = new FFN; + model1->Add(8); + model1->Add(false, true, 1); + model1->Add(); + + // Check whether copy constructor is working or not. + CheckCopyFunction(model, trainData, trainLabels); + + // Check whether move constructor is working or not. + CheckMoveFunction(model1, trainData, trainLabels, 1); +} +*/ + +/** + * Check whether copying and moving network with Reparametrization is working or not. + */ +// TEST_CASE("CheckCopyMovingReparametrizationNetworkTest", +// "[FeedForwardNetworkTest]") +// { +// // Load the dataset. +// arma::mat trainData; +// data::Load("thyroid_train.csv", trainData, true); +// +// // Normalize labels to [0, 2]. +// arma::mat trainLabels = trainData.row(trainData.n_rows - 1) - 1; +// trainData.shed_row(trainData.n_rows - 1); +// +// /* +// * Construct a feed forward network with trainData.n_rows input nodes, +// * followed by a linear layer and then a reparametrization layer. +// */ +// +// FFN *model = new FFN; +// model->Add >(trainData.n_rows, 8); +// model->Add >(4, false, true, 1); +// model->Add >(); +// +// FFN *model1 = new FFN; +// model1->Add >(trainData.n_rows, 8); +// model1->Add >(4, false, true, 1); +// model1->Add >(); +// +// // Check whether copy constructor is working or not. +// CheckCopyFunction<>(model, trainData, trainLabels, 1); +// +// // Check whether move constructor is working or not. +// CheckMoveFunction<>(model1, trainData, trainLabels, 1); +// } + +/** + * Check whether copying and moving network with Reparametrization is working or not. + */ +// TEST_CASE("CheckCopyMovingReparametrizationNetworkTestNoBias", +// "[FeedForwardNetworkTest]") +// { +// // Load the dataset. +// arma::mat trainData; +// data::Load("thyroid_train.csv", trainData, true); +// +// // Normalize labels to [0, 2]. +// arma::mat trainLabels = trainData.row(trainData.n_rows - 1) - 1; +// trainData.shed_row(trainData.n_rows - 1); +// +// /* +// * Construct a feed forward network with trainData.n_rows input nodes, +// * followed by a linear layer and then a reparametrization layer. +// */ +// +// FFN *model = new FFN; +// model->Add >(trainData.n_rows, 8); +// model->Add >(4, false, true, 1); +// model->Add >(); +// +// FFN *model1 = new FFN; +// model1->Add >(trainData.n_rows, 8); +// model1->Add >(4, false, true, 1); +// model1->Add >(); +// +// // Check whether copy constructor is working or not. +// CheckCopyFunction<>(model, trainData, trainLabels, 1); +// +// // Check whether move constructor is working or not. +// CheckMoveFunction<>(model1, trainData, trainLabels, 1); +// } + +/** + * Train the highway network on a larger dataset. + * +TEST_CASE("HighwayNetworkTest", "[FeedForwardNetworkTest]") +{ + arma::mat dataset; + dataset.load("mnist_first250_training_4s_and_9s.arm"); + + // Normalize each point since these are images. + for (size_t i = 0; i < dataset.n_cols; ++i) + dataset.col(i) /= norm(dataset.col(i), 2); + + arma::mat labels = arma::zeros(1, dataset.n_cols); + labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); + + FFN model; + model.Add(10); + Highway* highway = new Highway(); + highway->Add(10); + highway->Add(); + model.Add(highway); // This takes ownership of the memory. + model.Add(2); + model.Add(); + TestNetwork(model, dataset, labels, dataset, labels, 10, 0.2); +}*/ + +/** + * Test that serialization works ok for PReLU. + */ +// TEST_CASE("PReLUSerializationTest", "[FeedForwardNetworkTest]") +// { +// // Load the dataset. +// arma::mat trainData; +// if (!data::Load("thyroid_train.csv", trainData)) +// FAIL("Cannot open thyroid_train.csv"); +// +// arma::mat trainLabels = trainData.row(trainData.n_rows - 1); +// trainData.shed_row(trainData.n_rows - 1); +// trainLabels -= 1; // The labels should be between 0 and numClasses - 1. +// +// arma::mat testData; +// if (!data::Load("thyroid_test.csv", testData)) +// FAIL("Cannot load dataset thyroid_test.csv"); +// +// arma::mat testLabels = testData.row(testData.n_rows - 1); +// testData.shed_row(testData.n_rows - 1); +// testLabels -= 1; // The labels should be between 0 and numClasses - 1. +// +// // Vanilla neural net with logistic activation function. +// // Because 92% of the patients are not hyperthyroid the neural +// // network must be significant better than 92%. +// FFN model; +// model.Add >(trainData.n_rows, 8); +// model.Add >(); +// model.Add >(); +// model.Add >(8, 3); +// model.Add >(); +// +// ens::RMSProp opt(0.01, 32, 0.88, 1e-8, trainData.n_cols /* 1 epoch */, -1); +// +// model.Train(trainData, trainLabels, opt); +// +// FFN xmlModel, jsonModel, binaryModel; +// xmlModel.Add>(10, 10); // Layer that will get removed. +// +// // Serialize into other models. +// SerializeObjectAll(model, xmlModel, jsonModel, binaryModel); +// +// arma::mat predictions, xmlPredictions, jsonPredictions, binaryPredictions; +// model.Predict(testData, predictions); +// xmlModel.Predict(testData, xmlPredictions); +// jsonModel.Predict(testData, jsonPredictions); +// jsonModel.Predict(testData, binaryPredictions); +// +// CheckMatrices(predictions, xmlPredictions, jsonPredictions, +// binaryPredictions); +// } + +/** + * Test if the custom layers work. The target is to see if the code compiles + * when the Train and Prediction are called. + */ +// TEST_CASE("CustomLayerTest", "[FeedForwardNetworkTest]") +// { +// // Load the dataset. +// arma::mat trainData; +// if (!data::Load("thyroid_train.csv", trainData)) +// FAIL("Cannot open thyroid_train.csv"); +// +// arma::mat trainLabels = trainData.row(trainData.n_rows - 1); +// trainData.shed_row(trainData.n_rows - 1); +// trainLabels -= 1; // The labels should be between 0 and numClasses - 1. +// +// arma::mat testData; +// if (!data::Load("thyroid_test.csv", testData)) +// FAIL("Cannot load dataset thyroid_test.csv"); +// +// arma::mat testLabels = testData.row(testData.n_rows - 1); +// testData.shed_row(testData.n_rows - 1); +// testLabels -= 1; // The labels should be between 0 and numClasses - 1. +// +// FFN > model; +// model.Add >(trainData.n_rows, 8); +// model.Add >(); +// model.Add >(8, 3); +// model.Add >(); +// +// ens::RMSProp opt(0.01, 32, 0.88, 1e-8, 15, -1); +// model.Train(trainData, trainLabels, opt); +// +// arma::mat predictionTemp; +// model.Predict(testData, predictionTemp); +// arma::mat prediction = arma::zeros(1, predictionTemp.n_cols); +// } diff --git a/src/mlpack/tests/ann/not_adapted/recurrent_network_test.cpp b/src/mlpack/tests/ann/not_adapted/recurrent_network_test.cpp new file mode 100644 index 0000000000..e13c3e88bb --- /dev/null +++ b/src/mlpack/tests/ann/not_adapted/recurrent_network_test.cpp @@ -0,0 +1,464 @@ +/** + * @file tests/recurrent_network_test.cpp + * @author Marcus Edel + * + * Tests the recurrent network. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include +#include +#include +#include + +#include + +#include "../catch.hpp" +#include "../serialization.hpp" + +using namespace mlpack; +using namespace mlpack::ann; +using namespace ens; +using namespace mlpack::math; + +/** + * Train the specified networks on the Derek D. Monner's distracted sequence + * recall task. + */ +/* TEST_CASE("LSTMDistractedSequenceRecallTest", "[RecurrentNetworkTest]") */ +/* { */ +/* DistractedSequenceRecallTestNetwork >(4, 8); */ +/* } */ + +/** + * Train the specified networks on the Derek D. Monner's distracted sequence + * recall task. + */ +/* TEST_CASE("FastLSTMDistractedSequenceRecallTest", "[RecurrentNetworkTest]") */ +/* { */ +/* DistractedSequenceRecallTestNetwork >(4, 8); */ +/* } */ + +/** + * Train the specified networks on the Derek D. Monner's distracted sequence + * recall task. + */ +/* TEST_CASE("GRUDistractedSequenceRecallTest", "[RecurrentNetworkTest]") */ +/* { */ +/* DistractedSequenceRecallTestNetwork >(4, 8); */ +/* } */ + +/** + * Ensure fast LSTMs work with larger batch sizes. + */ +//TEST_CASE("FastLSTMBatchSizeTest", "[RecurrentNetworkTest]") +//{ +// BatchSizeTest>(); +//} + +/** + * Ensure GRUs work with larger batch sizes. + */ +//TEST_CASE("GRUBatchSizeTest", "[RecurrentNetworkTest]") +//{ +// BatchSizeTest>(); +//} + +/** + * Make sure the RNN can be properly serialized. + * +TEST_CASE("RNNSerializationTest", "[RecurrentNetworkTest]") +{ + const size_t rho = 10; + + // Generate 12 (2 * 6) noisy sines. A single sine contains rho + // points/features. + arma::cube input; + arma::mat labelsTemp; + GenerateNoisySines(input, labelsTemp, rho, 6); + + arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); + for (size_t i = 0; i < labelsTemp.n_cols; ++i) + { + const int value = arma::as_scalar(arma::find( + arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; + labels.tube(0, i).fill(value); + } + + /** + * Construct a network with 1 input unit, 4 hidden units and 10 output + * units. The hidden layer is connected to itself. The network structure + * looks like: + * + * Input Hidden Output + * Layer(1) Layer(4) Layer(10) + * +-----+ +-----+ +-----+ + * | | | | | | + * | +------>| +------>| | + * | | ..>| | | | + * +-----+ . +--+--+ +-----+ + * . . + * . . + * ....... + * + Add<> add(4); + Linear<> lookup(1, 4); + SigmoidLayer<> sigmoidLayer; + Linear<> linear(4, 4); + Recurrent<>* recurrent = new Recurrent<>(add, lookup, linear, + sigmoidLayer, rho); + + RNN<> model(rho); + model.Add >(); + model.Add(recurrent); + model.Add >(4, 10); + model.Add >(); + + StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch *, -100); + model.Train(input, labels, opt); + + // Serialize the network. + RNN<> xmlModel(1), jsonModel(3), binaryModel(5); + SerializeObjectAll(model, xmlModel, jsonModel, binaryModel); + + // Take predictions, check the output. + arma::cube prediction, xmlPrediction, jsonPrediction, binaryPrediction; + model.Predict(input, prediction); + xmlModel.Predict(input, xmlPrediction); + jsonModel.Predict(input, jsonPrediction); + binaryModel.Predict(input, binaryPrediction); + + CheckMatrices(prediction, xmlPrediction, jsonPrediction, binaryPrediction); +} +*/ + +/** + * Train the BRNN on a larger dataset. + * +TEST_CASE("SequenceClassificationBRNNTest", "[RecurrentNetworkTest]") +{ + // Using same test for RNN below. + size_t successes = 0; + const size_t rho = 10; + + for (size_t trial = 0; trial < 6; ++trial) + { + // Generate 12 (2 * 6) noisy sines. A single sine contains rho + // points/features. + arma::cube input; + arma::mat labelsTemp; + GenerateNoisySines(input, labelsTemp, rho, 6); + + arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); + for (size_t i = 0; i < labelsTemp.n_cols; ++i) + { + const int value = arma::as_scalar(arma::find( + arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)); + labels.tube(0, i).fill(value); + } + + Add<> add(4); + Linear<> lookup(1, 4); + SigmoidLayer<> sigmoidLayer; + Linear<> linear(4, 4); + Recurrent<>* recurrent = new Recurrent<>( + add, lookup, linear, sigmoidLayer, rho); + + BRNN<> model(rho); + model.Add >(); + model.Add(recurrent); + model.Add >(4, 5); + + StandardSGD opt(0.1, 1, 500 * input.n_cols, -100); + model.Train(input, labels, opt); + INFO("Training over"); + arma::cube prediction; + model.Predict(input, prediction); + INFO("Prediction over"); + + size_t error = 0; + for (size_t i = 0; i < prediction.n_cols; ++i) + { + const int predictionValue = arma::as_scalar(arma::find( + arma::max(prediction.slice(rho - 1).col(i)) == + prediction.slice(rho - 1).col(i), 1)); + + const int targetValue = arma::as_scalar(arma::find( + arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)); + + if (predictionValue == targetValue) + { + error++; + } + } + + double classificationError = 1 - double(error) / prediction.n_cols; + INFO(classificationError); + if (classificationError <= 0.2) + { + ++successes; + break; + } + } + + REQUIRE(successes >= 1); +} +*/ + +/** + * Train the vanilla network on a larger dataset. + * +TEST_CASE("SequenceClassificationTest", "[RecurrentNetworkTest]") +{ + // It isn't guaranteed that the recurrent network will converge in the + // specified number of iterations using random weights. If this works 1 of 6 + // times, I'm fine with that. All I want to know is that the network is able + // to escape from local minima and to solve the task. + size_t successes = 0; + const size_t rho = 10; + + for (size_t trial = 0; trial < 6; ++trial) + { + // Generate 12 (2 * 6) noisy sines. A single sine contains rho + // points/features. + arma::cube input; + arma::mat labelsTemp; + GenerateNoisySines(input, labelsTemp, rho, 6); + + arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); + for (size_t i = 0; i < labelsTemp.n_cols; ++i) + { + const int value = arma::as_scalar(arma::find( + arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)); + labels.tube(0, i).fill(value); + } + + /** + * Construct a network with 1 input unit, 4 hidden units and 10 output + * units. The hidden layer is connected to itself. The network structure + * looks like: + * + * Input Hidden Output + * Layer(1) Layer(4) Layer(10) + * +-----+ +-----+ +-----+ + * | | | | | | + * | +------>| +------>| | + * | | ..>| | | | + * +-----+ . +--+--+ +-----+ + * . . + * . . + * ....... + * + Add<> add(4); + Linear<> lookup(1, 4); + SigmoidLayer<> sigmoidLayer; + Linear<> linear(4, 4); + Recurrent<>* recurrent = new Recurrent<>( + add, lookup, linear, sigmoidLayer, rho); + + RNN<> model(rho); + model.Add >(); + model.Add(recurrent); + model.Add >(4, 10); + model.Add >(); + + StandardSGD opt(0.1, 1, 500 * input.n_cols, -100); + model.Train(input, labels, opt); + + arma::cube prediction; + model.Predict(input, prediction); + + size_t error = 0; + for (size_t i = 0; i < prediction.n_cols; ++i) + { + const int predictionValue = arma::as_scalar(arma::find( + arma::max(prediction.slice(rho - 1).col(i)) == + prediction.slice(rho - 1).col(i), 1)); + + const int targetValue = arma::as_scalar(arma::find( + arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)); + + if (predictionValue == targetValue) + { + error++; + } + } + + double classificationError = 1 - double(error) / prediction.n_cols; + if (classificationError <= 0.2) + { + ++successes; + break; + } + } + + REQUIRE(successes >= 1); +} +*/ + +/** + * Test that RNN::Train() returns finite objective value. + * +TEST_CASE("RNNTrainReturnObjective", "[RecurrentNetworkTest]") +{ + const size_t rho = 10; + + // Generate 12 (2 * 6) noisy sines. A single sine contains rho + // points/features. + arma::cube input; + arma::mat labelsTemp; + GenerateNoisySines(input, labelsTemp, rho, 6); + + arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); + for (size_t i = 0; i < labelsTemp.n_cols; ++i) + { + const int value = arma::as_scalar(arma::find( + arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)); + labels.tube(0, i).fill(value); + } + + /** + * Construct a network with 1 input unit, 4 hidden units and 10 output + * units. The hidden layer is connected to itself. The network structure + * looks like: + * + * Input Hidden Output + * Layer(1) Layer(4) Layer(10) + * +-----+ +-----+ +-----+ + * | | | | | | + * | +------>| +------>| | + * | | ..>| | | | + * +-----+ . +--+--+ +-----+ + * . . + * . . + * ....... + * + Add<> add(4); + Linear<> lookup(1, 4); + SigmoidLayer<> sigmoidLayer; + Linear<> linear(4, 4); + Recurrent<>* recurrent = new Recurrent<>(add, lookup, linear, + sigmoidLayer, rho); + + RNN<> model(rho); + model.Add >(); + model.Add(recurrent); + model.Add >(4, 10); + model.Add >(); + + StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch *, -100); + double objVal = model.Train(input, labels, opt); + + REQUIRE(std::isfinite(objVal) == true); +} +*/ + +/** + * Test that BRNN::Train() returns finite objective value. + * +TEST_CASE("BRNNTrainReturnObjective", "[RecurrentNetworkTest]") +{ + const size_t rho = 10; + + arma::cube input; + arma::mat labelsTemp; + GenerateNoisySines(input, labelsTemp, rho, 6); + + arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); + for (size_t i = 0; i < labelsTemp.n_cols; ++i) + { + const int value = arma::as_scalar(arma::find( + arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)); + labels.tube(0, i).fill(value); + } + + Add<> add(4); + Linear<> lookup(1, 4); + SigmoidLayer<> sigmoidLayer; + Linear<> linear(4, 4); + Recurrent<>* recurrent = new Recurrent<>( + add, lookup, linear, sigmoidLayer, rho); + + BRNN<> model(rho); + model.Add >(); + model.Add(recurrent); + model.Add >(4, 5); + + StandardSGD opt(0.1, 1, 500 * input.n_cols, -100); + double objVal = model.Train(input, labels, opt); + INFO("Training over"); + + // Test that BRNN::Train() returns finite objective value. + REQUIRE(std::isfinite(objVal) == true); +} +*/ + +/** + * Test to make sure that an error is thrown when input with + * wrong input shape is provided to a RNN. + * +TEST_CASE("RNNCheckInputShapeTest", "[RecurrentNetworkTest]") +{ + const size_t rho = 10; + + // Generate 12 (2 * 6) noisy sines. A single sine contains rho + // points/features. + arma::cube input; + arma::mat labelsTemp; + GenerateNoisySines(input, labelsTemp, rho, 6); + + arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); + for (size_t i = 0; i < labelsTemp.n_cols; ++i) + { + const int value = arma::as_scalar(arma::find( + arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; + labels.tube(0, i).fill(value); + } + + /** + * Construct a network with 1 input unit, 4 hidden units and 10 output + * units. The hidden layer is connected to itself. The network structure + * looks like: + * + * Input Hidden Output + * Layer(1) Layer(4) Layer(10) + * +-----+ +-----+ +-----+ + * | | | | | | + * | +------>| +------>| | + * | | ..>| | | | + * +-----+ . +--+--+ +-----+ + * . . + * . . + * ....... + * + Add<> add(4); + // Purposely providing wrong input shape of 3. + // The correct input shape is 1. + Linear<> lookup(3, 4); + SigmoidLayer<> sigmoidLayer; + Linear<> linear(4, 4); + Recurrent<>* recurrent = new Recurrent<>(add, lookup, linear, + sigmoidLayer, rho); + + RNN<> model(rho); + model.Add >(); + model.Add(recurrent); + model.Add >(4, 10); + model.Add >(); + + std::string expectedMsg = "RNN<>::Train(): "; + expectedMsg += "the first layer of the network expects "; + expectedMsg += std::to_string(3) + " elements, "; + expectedMsg += "but the input has " + std::to_string(1) + " dimensions! "; + + StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch *, -100); + + REQUIRE_THROWS_AS(model.Train(input, labels, opt), std::logic_error); +} +*/ diff --git a/src/mlpack/tests/ann/recurrent_network_test.cpp b/src/mlpack/tests/ann/recurrent_network_test.cpp index 1d12db8ee0..c2cd974d0a 100644 --- a/src/mlpack/tests/ann/recurrent_network_test.cpp +++ b/src/mlpack/tests/ann/recurrent_network_test.cpp @@ -149,34 +149,6 @@ void GenerateDistractedSequence(arma::mat& input, arma::mat& output) output.reshape(output.n_elem, 1); } - -/** - * Train the specified networks on the Derek D. Monner's distracted sequence - * recall task. - */ -/* TEST_CASE("LSTMDistractedSequenceRecallTest", "[RecurrentNetworkTest]") */ -/* { */ -/* DistractedSequenceRecallTestNetwork >(4, 8); */ -/* } */ - -/** - * Train the specified networks on the Derek D. Monner's distracted sequence - * recall task. - */ -/* TEST_CASE("FastLSTMDistractedSequenceRecallTest", "[RecurrentNetworkTest]") */ -/* { */ -/* DistractedSequenceRecallTestNetwork >(4, 8); */ -/* } */ - -/** - * Train the specified networks on the Derek D. Monner's distracted sequence - * recall task. - */ -/* TEST_CASE("GRUDistractedSequenceRecallTest", "[RecurrentNetworkTest]") */ -/* { */ -/* DistractedSequenceRecallTestNetwork >(4, 8); */ -/* } */ - /** * Create a simple recurrent neural network for the noisy sines task, and * require that it produces the exact same network for a few batch sizes. @@ -241,254 +213,6 @@ TEST_CASE("LSTMBatchSizeTest", "[RecurrentNetworkTest]") BatchSizeTest(); } -/** - * Ensure fast LSTMs work with larger batch sizes. - */ -//TEST_CASE("FastLSTMBatchSizeTest", "[RecurrentNetworkTest]") -//{ -// BatchSizeTest>(); -//} - -/** - * Ensure GRUs work with larger batch sizes. - */ -//TEST_CASE("GRUBatchSizeTest", "[RecurrentNetworkTest]") -//{ -// BatchSizeTest>(); -//} - -/** - * Make sure the RNN can be properly serialized. - * -TEST_CASE("RNNSerializationTest", "[RecurrentNetworkTest]") -{ - const size_t rho = 10; - - // Generate 12 (2 * 6) noisy sines. A single sine contains rho - // points/features. - arma::cube input; - arma::mat labelsTemp; - GenerateNoisySines(input, labelsTemp, rho, 6); - - arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); - for (size_t i = 0; i < labelsTemp.n_cols; ++i) - { - const int value = arma::as_scalar(arma::find( - arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; - labels.tube(0, i).fill(value); - } - - /** - * Construct a network with 1 input unit, 4 hidden units and 10 output - * units. The hidden layer is connected to itself. The network structure - * looks like: - * - * Input Hidden Output - * Layer(1) Layer(4) Layer(10) - * +-----+ +-----+ +-----+ - * | | | | | | - * | +------>| +------>| | - * | | ..>| | | | - * +-----+ . +--+--+ +-----+ - * . . - * . . - * ....... - * - Add<> add(4); - Linear<> lookup(1, 4); - SigmoidLayer<> sigmoidLayer; - Linear<> linear(4, 4); - Recurrent<>* recurrent = new Recurrent<>(add, lookup, linear, - sigmoidLayer, rho); - - RNN<> model(rho); - model.Add >(); - model.Add(recurrent); - model.Add >(4, 10); - model.Add >(); - - StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch *, -100); - model.Train(input, labels, opt); - - // Serialize the network. - RNN<> xmlModel(1), jsonModel(3), binaryModel(5); - SerializeObjectAll(model, xmlModel, jsonModel, binaryModel); - - // Take predictions, check the output. - arma::cube prediction, xmlPrediction, jsonPrediction, binaryPrediction; - model.Predict(input, prediction); - xmlModel.Predict(input, xmlPrediction); - jsonModel.Predict(input, jsonPrediction); - binaryModel.Predict(input, binaryPrediction); - - CheckMatrices(prediction, xmlPrediction, jsonPrediction, binaryPrediction); -} -*/ - -/** - * Train the BRNN on a larger dataset. - * -TEST_CASE("SequenceClassificationBRNNTest", "[RecurrentNetworkTest]") -{ - // Using same test for RNN below. - size_t successes = 0; - const size_t rho = 10; - - for (size_t trial = 0; trial < 6; ++trial) - { - // Generate 12 (2 * 6) noisy sines. A single sine contains rho - // points/features. - arma::cube input; - arma::mat labelsTemp; - GenerateNoisySines(input, labelsTemp, rho, 6); - - arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); - for (size_t i = 0; i < labelsTemp.n_cols; ++i) - { - const int value = arma::as_scalar(arma::find( - arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)); - labels.tube(0, i).fill(value); - } - - Add<> add(4); - Linear<> lookup(1, 4); - SigmoidLayer<> sigmoidLayer; - Linear<> linear(4, 4); - Recurrent<>* recurrent = new Recurrent<>( - add, lookup, linear, sigmoidLayer, rho); - - BRNN<> model(rho); - model.Add >(); - model.Add(recurrent); - model.Add >(4, 5); - - StandardSGD opt(0.1, 1, 500 * input.n_cols, -100); - model.Train(input, labels, opt); - INFO("Training over"); - arma::cube prediction; - model.Predict(input, prediction); - INFO("Prediction over"); - - size_t error = 0; - for (size_t i = 0; i < prediction.n_cols; ++i) - { - const int predictionValue = arma::as_scalar(arma::find( - arma::max(prediction.slice(rho - 1).col(i)) == - prediction.slice(rho - 1).col(i), 1)); - - const int targetValue = arma::as_scalar(arma::find( - arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)); - - if (predictionValue == targetValue) - { - error++; - } - } - - double classificationError = 1 - double(error) / prediction.n_cols; - INFO(classificationError); - if (classificationError <= 0.2) - { - ++successes; - break; - } - } - - REQUIRE(successes >= 1); -} -*/ - -/** - * Train the vanilla network on a larger dataset. - * -TEST_CASE("SequenceClassificationTest", "[RecurrentNetworkTest]") -{ - // It isn't guaranteed that the recurrent network will converge in the - // specified number of iterations using random weights. If this works 1 of 6 - // times, I'm fine with that. All I want to know is that the network is able - // to escape from local minima and to solve the task. - size_t successes = 0; - const size_t rho = 10; - - for (size_t trial = 0; trial < 6; ++trial) - { - // Generate 12 (2 * 6) noisy sines. A single sine contains rho - // points/features. - arma::cube input; - arma::mat labelsTemp; - GenerateNoisySines(input, labelsTemp, rho, 6); - - arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); - for (size_t i = 0; i < labelsTemp.n_cols; ++i) - { - const int value = arma::as_scalar(arma::find( - arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)); - labels.tube(0, i).fill(value); - } - - /** - * Construct a network with 1 input unit, 4 hidden units and 10 output - * units. The hidden layer is connected to itself. The network structure - * looks like: - * - * Input Hidden Output - * Layer(1) Layer(4) Layer(10) - * +-----+ +-----+ +-----+ - * | | | | | | - * | +------>| +------>| | - * | | ..>| | | | - * +-----+ . +--+--+ +-----+ - * . . - * . . - * ....... - * - Add<> add(4); - Linear<> lookup(1, 4); - SigmoidLayer<> sigmoidLayer; - Linear<> linear(4, 4); - Recurrent<>* recurrent = new Recurrent<>( - add, lookup, linear, sigmoidLayer, rho); - - RNN<> model(rho); - model.Add >(); - model.Add(recurrent); - model.Add >(4, 10); - model.Add >(); - - StandardSGD opt(0.1, 1, 500 * input.n_cols, -100); - model.Train(input, labels, opt); - - arma::cube prediction; - model.Predict(input, prediction); - - size_t error = 0; - for (size_t i = 0; i < prediction.n_cols; ++i) - { - const int predictionValue = arma::as_scalar(arma::find( - arma::max(prediction.slice(rho - 1).col(i)) == - prediction.slice(rho - 1).col(i), 1)); - - const int targetValue = arma::as_scalar(arma::find( - arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)); - - if (predictionValue == targetValue) - { - error++; - } - } - - double classificationError = 1 - double(error) / prediction.n_cols; - if (classificationError <= 0.2) - { - ++successes; - break; - } - } - - REQUIRE(successes >= 1); -} -*/ - /** * @brief Generates noisy sine wave and outputs the data and the labels that * can be used directly for training and testing with RNN. @@ -622,103 +346,6 @@ TEST_CASE("MultiTimestepTest", "[RecurrentNetworkTest]") REQUIRE(err <= 0.025); } -/** - * Test that RNN::Train() returns finite objective value. - * -TEST_CASE("RNNTrainReturnObjective", "[RecurrentNetworkTest]") -{ - const size_t rho = 10; - - // Generate 12 (2 * 6) noisy sines. A single sine contains rho - // points/features. - arma::cube input; - arma::mat labelsTemp; - GenerateNoisySines(input, labelsTemp, rho, 6); - - arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); - for (size_t i = 0; i < labelsTemp.n_cols; ++i) - { - const int value = arma::as_scalar(arma::find( - arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)); - labels.tube(0, i).fill(value); - } - - /** - * Construct a network with 1 input unit, 4 hidden units and 10 output - * units. The hidden layer is connected to itself. The network structure - * looks like: - * - * Input Hidden Output - * Layer(1) Layer(4) Layer(10) - * +-----+ +-----+ +-----+ - * | | | | | | - * | +------>| +------>| | - * | | ..>| | | | - * +-----+ . +--+--+ +-----+ - * . . - * . . - * ....... - * - Add<> add(4); - Linear<> lookup(1, 4); - SigmoidLayer<> sigmoidLayer; - Linear<> linear(4, 4); - Recurrent<>* recurrent = new Recurrent<>(add, lookup, linear, - sigmoidLayer, rho); - - RNN<> model(rho); - model.Add >(); - model.Add(recurrent); - model.Add >(4, 10); - model.Add >(); - - StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch *, -100); - double objVal = model.Train(input, labels, opt); - - REQUIRE(std::isfinite(objVal) == true); -} -*/ - -/** - * Test that BRNN::Train() returns finite objective value. - * -TEST_CASE("BRNNTrainReturnObjective", "[RecurrentNetworkTest]") -{ - const size_t rho = 10; - - arma::cube input; - arma::mat labelsTemp; - GenerateNoisySines(input, labelsTemp, rho, 6); - - arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); - for (size_t i = 0; i < labelsTemp.n_cols; ++i) - { - const int value = arma::as_scalar(arma::find( - arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)); - labels.tube(0, i).fill(value); - } - - Add<> add(4); - Linear<> lookup(1, 4); - SigmoidLayer<> sigmoidLayer; - Linear<> linear(4, 4); - Recurrent<>* recurrent = new Recurrent<>( - add, lookup, linear, sigmoidLayer, rho); - - BRNN<> model(rho); - model.Add >(); - model.Add(recurrent); - model.Add >(4, 5); - - StandardSGD opt(0.1, 1, 500 * input.n_cols, -100); - double objVal = model.Train(input, labels, opt); - INFO("Training over"); - - // Test that BRNN::Train() returns finite objective value. - REQUIRE(std::isfinite(objVal) == true); -} -*/ - /** * Test that RNN::Train() does not give an error for large rho. */ @@ -781,70 +408,6 @@ TEST_CASE("LargeRhoValueRnnTest", "[RecurrentNetworkTest]") INFO("Training over"); } -/** - * Test to make sure that an error is thrown when input with - * wrong input shape is provided to a RNN. - * -TEST_CASE("RNNCheckInputShapeTest", "[RecurrentNetworkTest]") -{ - const size_t rho = 10; - - // Generate 12 (2 * 6) noisy sines. A single sine contains rho - // points/features. - arma::cube input; - arma::mat labelsTemp; - GenerateNoisySines(input, labelsTemp, rho, 6); - - arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); - for (size_t i = 0; i < labelsTemp.n_cols; ++i) - { - const int value = arma::as_scalar(arma::find( - arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; - labels.tube(0, i).fill(value); - } - - /** - * Construct a network with 1 input unit, 4 hidden units and 10 output - * units. The hidden layer is connected to itself. The network structure - * looks like: - * - * Input Hidden Output - * Layer(1) Layer(4) Layer(10) - * +-----+ +-----+ +-----+ - * | | | | | | - * | +------>| +------>| | - * | | ..>| | | | - * +-----+ . +--+--+ +-----+ - * . . - * . . - * ....... - * - Add<> add(4); - // Purposely providing wrong input shape of 3. - // The correct input shape is 1. - Linear<> lookup(3, 4); - SigmoidLayer<> sigmoidLayer; - Linear<> linear(4, 4); - Recurrent<>* recurrent = new Recurrent<>(add, lookup, linear, - sigmoidLayer, rho); - - RNN<> model(rho); - model.Add >(); - model.Add(recurrent); - model.Add >(4, 10); - model.Add >(); - - std::string expectedMsg = "RNN<>::Train(): "; - expectedMsg += "the first layer of the network expects "; - expectedMsg += std::to_string(3) + " elements, "; - expectedMsg += "but the input has " + std::to_string(1) + " dimensions! "; - - StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch *, -100); - - REQUIRE_THROWS_AS(model.Train(input, labels, opt), std::logic_error); -} -*/ - /** * Test that a simple RNN with no recurrent components behaves the same as an * FFN. From b09775dc1ca5fe7c0d42b450c6b32b7880430128 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sun, 24 Apr 2022 10:03:23 -0400 Subject: [PATCH 04/39] Split layer tests into their own files. --- src/mlpack/tests/CMakeLists.txt | 3 +- src/mlpack/tests/ann/ann_layer_test.cpp | 795 ------------------ src/mlpack/tests/ann/ann_test_tools.hpp | 2 + src/mlpack/tests/ann/layer/alpha_dropout.cpp | 116 +++ src/mlpack/tests/ann/layer/convolution.cpp | 235 ++++++ src/mlpack/tests/ann/layer/dropout.cpp | 105 +++ src/mlpack/tests/ann/layer/linear3d.cpp | 142 ++++ src/mlpack/tests/ann/layer/linear_no_bias.cpp | 116 +++ src/mlpack/tests/ann/layer/log_softmax.cpp | 49 ++ src/mlpack/tests/ann/layer/max_pooling.cpp | 118 +++ src/mlpack/tests/ann/layer/padding.cpp | 87 ++ src/mlpack/tests/ann/layer_test.cpp | 21 + 12 files changed, 993 insertions(+), 796 deletions(-) delete mode 100644 src/mlpack/tests/ann/ann_layer_test.cpp create mode 100644 src/mlpack/tests/ann/layer/alpha_dropout.cpp create mode 100644 src/mlpack/tests/ann/layer/convolution.cpp create mode 100644 src/mlpack/tests/ann/layer/dropout.cpp create mode 100644 src/mlpack/tests/ann/layer/linear3d.cpp create mode 100644 src/mlpack/tests/ann/layer/linear_no_bias.cpp create mode 100644 src/mlpack/tests/ann/layer/log_softmax.cpp create mode 100644 src/mlpack/tests/ann/layer/max_pooling.cpp create mode 100644 src/mlpack/tests/ann/layer/padding.cpp create mode 100644 src/mlpack/tests/ann/layer_test.cpp diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index ff11ee9386..c5f25ca1a6 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -6,7 +6,8 @@ add_executable(mlpack_test # Tests for neural network code (and reinforcement learning code). ann/activation_functions_test.cpp ann/ann_dist_test.cpp - ann/ann_layer_test.cpp + # This includes all files in ann/layer/. + ann/layer_test.cpp ann/ann_regularizer_test.cpp ann/ann_test_tools.hpp ann/convolutional_network_test.cpp diff --git a/src/mlpack/tests/ann/ann_layer_test.cpp b/src/mlpack/tests/ann/ann_layer_test.cpp deleted file mode 100644 index 8e911501e2..0000000000 --- a/src/mlpack/tests/ann/ann_layer_test.cpp +++ /dev/null @@ -1,795 +0,0 @@ -/** - * @file tests/ann_layer_test.cpp - * @author Marcus Edel - * @author Praveen Ch - * - * Tests the ann layer modules. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../test_catch_tools.hpp" -#include "../catch.hpp" -#include "../serialization.hpp" -#include "ann_test_tools.hpp" - -using namespace mlpack; -using namespace mlpack::ann; - -/** - * Simple dropout module test. - */ -TEST_CASE("SimpleDropoutLayerTest", "[ANNLayerTest]") -{ - // Initialize the probability of setting a value to zero. - const double p = 0.2; - - // Initialize the input parameter. - arma::mat input(1000, 1); - input.fill(1 - p); - - Dropout module(p); - module.Training() = true; - - // Test the Forward function. - arma::mat output; - module.Forward(input, output); - REQUIRE(arma::as_scalar(arma::abs(arma::mean(output) - (1 - p))) <= 0.05); - - // Test the Backward function. - arma::mat delta; - module.Backward(input, input, delta); - REQUIRE(arma::as_scalar(arma::abs(arma::mean(delta) - (1 - p))) <= 0.05); - - // Test the Forward function. - module.Training() = false; - module.Forward(input, output); - REQUIRE(arma::accu(input) == arma::accu(output)); -} - -/** - * Perform dropout x times using ones as input, sum the number of ones and - * validate that the layer is producing approximately the correct number of - * ones. - */ -TEST_CASE("DropoutProbabilityTest", "[ANNLayerTest]") -{ - arma::mat input = arma::ones(1500, 1); - const size_t iterations = 10; - - double probability[5] = { 0.1, 0.3, 0.4, 0.7, 0.8 }; - for (size_t trial = 0; trial < 5; ++trial) - { - double nonzeroCount = 0; - for (size_t i = 0; i < iterations; ++i) - { - Dropout module(probability[trial]); - module.Training() = true; - - arma::mat output; - module.Forward(input, output); - - // Return a column vector containing the indices of elements of X that - // are non-zero, we just need the number of non-zero values. - arma::uvec nonzero = arma::find(output); - nonzeroCount += nonzero.n_elem; - } - const double expected = input.n_elem * (1 - probability[trial]) * - iterations; - const double error = fabs(nonzeroCount - expected) / expected; - - REQUIRE(error <= 0.15); - } -} - -/* - * Perform dropout with probability 1 - p where p = 0, means no dropout. - */ -TEST_CASE("NoDropoutTest", "[ANNLayerTest]") -{ - arma::mat input = arma::ones(1500, 1); - Dropout module(0); - module.Training() = true; - - arma::mat output; - module.Forward(input, output); - - REQUIRE(arma::accu(output) == arma::accu(input)); -} - -/* - * Perform test to check whether mean and variance remain nearly same - * after AlphaDropout. - */ -TEST_CASE("SimpleAlphaDropoutLayerTest", "[ANNLayerTest]") -{ - // Initialize the probability of setting a value to alphaDash. - const double p = 0.2; - - // Initialize the input parameter having a mean nearabout 0 - // and variance nearabout 1. - arma::mat input = arma::randn(1000, 1); - - AlphaDropout module(p); - module.Training() = true; - - // Test the Forward function when training phase. - arma::mat output(arma::size(input)); - module.Forward(input, output); - // Check whether mean remains nearly same. - REQUIRE(arma::as_scalar(arma::abs(arma::mean(input) - arma::mean(output))) <= - 0.1); - - // Check whether variance remains nearly same. - REQUIRE(arma::as_scalar(arma::abs(arma::var(input) - arma::var(output))) <= - 0.1); - - // Test the Backward function when training phase. - arma::mat delta; - module.Backward(input, input, delta); - REQUIRE(arma::as_scalar(arma::abs(arma::mean(delta) - 0)) <= 0.05); - - // Test the Forward function when testing phase. - module.Training() = false; - module.Forward(input, output); - REQUIRE(arma::accu(input) == arma::accu(output)); -} - -/** - * Perform AlphaDropout x times using ones as input, sum the number of ones - * and validate that the layer is producing approximately the correct number - * of ones. - */ -TEST_CASE("AlphaDropoutProbabilityTest", "[ANNLayerTest]") -{ - arma::mat input = arma::ones(1500, 1); - const size_t iterations = 10; - - double probability[5] = { 0.1, 0.3, 0.4, 0.7, 0.8 }; - for (size_t trial = 0; trial < 5; ++trial) - { - double nonzeroCount = 0; - for (size_t i = 0; i < iterations; ++i) - { - AlphaDropout module(probability[trial]); - module.Training() = true; - - arma::mat output(arma::size(input)); - module.Forward(input, output); - - // Return a column vector containing the indices of elements of X - // that are not alphaDash, we just need the number of - // nonAlphaDash values. - arma::uvec nonAlphaDash = arma::find(module.Mask()); - nonzeroCount += nonAlphaDash.n_elem; - } - - const double expected = input.n_elem * (1-probability[trial]) * iterations; - - const double error = fabs(nonzeroCount - expected) / expected; - - REQUIRE(error <= 0.15); - } -} - -/** - * Perform AlphaDropout with probability 1 - p where p = 0, - * means no AlphaDropout. - */ -TEST_CASE("NoAlphaDropoutTest", "[ANNLayerTest]") -{ - arma::mat input = arma::ones(1500, 1); - AlphaDropout module(0); - module.Training() = false; - - arma::mat output; - module.Forward(input, output); - - REQUIRE(arma::accu(output) == arma::accu(input)); -} - -/** - * Simple Linear3D layer test. - */ -TEST_CASE("SimpleLinear3DLayerTest", "[ANNLayerTest]") -{ - const size_t inSize = 4; - const size_t outSize = 1; - const size_t nPoints = 2; - const size_t batchSize = 1; - arma::mat input, output, delta; - - // Create a Linear3D layer outside of a network, and then set its memory. - Linear3D module(outSize); - module.InputDimensions() = std::vector({ 4, 2 }); - module.ComputeOutputDimensions(); - arma::mat weights(module.WeightSize(), 1); - module.SetWeights(weights.memptr()); - - module.Parameters().randu(); - - // Test the Forward function. - input = arma::zeros(inSize * nPoints, batchSize); - output.set_size(outSize * nPoints, batchSize); - module.Forward(input, output); - REQUIRE(arma::accu(module.Bias()) - == Approx(arma::accu(output) / (nPoints * batchSize)).epsilon(1e-3)); - - // Test the Backward function. - delta.set_size(input.n_rows, input.n_cols); - output.zeros(); - module.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 0); -} - -/** - * Jacobian Linear3D module test. - */ -TEST_CASE("JacobianLinear3DLayerTest", "[ANNLayerTest]") -{ - for (size_t i = 0; i < 5; ++i) - { - const size_t inSize = math::RandInt(2, 10); - const size_t outSize = math::RandInt(2, 10); - const size_t nPoints = math::RandInt(2, 10); - const size_t batchSize = 1; - - arma::mat input; - input.set_size(inSize * nPoints, batchSize); - - // Create a Linear3D layer outside a network and initialize its memory. - Linear3D module(outSize); - module.InputDimensions() = std::vector({ inSize, nPoints }); - module.ComputeOutputDimensions(); - arma::mat weights(module.WeightSize(), 1); - module.SetWeights(weights.memptr()); - - module.Parameters().randu(); - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -} - -/** - * Simple Gradient test for Linear3D layer. - */ -TEST_CASE("GradientLinear3DLayerTest", "[ANNLayerTest]") -{ - // Linear function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - inSize(4), - outSize(1), - nPoints(2), - batchSize(4) - { - input = arma::randu(inSize * nPoints, batchSize); - target = arma::zeros(outSize * nPoints, batchSize); - target(0, 0) = 1; - target(0, 3) = 1; - target(1, 1) = 1; - target(1, 2) = 1; - - model = new FFN(); - model->ResetData(input, target); - model->Add(outSize); - model->InputDimensions() = std::vector{ 4, 2 }; - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - const size_t inSize; - const size_t outSize; - const size_t nPoints; - const size_t batchSize; - } function; - - REQUIRE(CheckGradient(function) <= 1e-7); -} - -/** - * Simple linear no bias module test. - */ -TEST_CASE("SimpleLinearNoBiasLayerTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta; - LinearNoBias module(10); - arma::mat weights(10 * 10, 1); - module.InputDimensions() = std::vector({ 10 }); - module.ComputeOutputDimensions(); - module.SetWeights(weights.memptr()); - - module.Parameters().randu(); - - // Test the Forward function. - input = arma::zeros(10, 1); - module.Forward(input, output); - REQUIRE(0 == arma::accu(output)); - - // Test the Backward function. - module.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 0); -} - -/** - * Simple padding layer test. - */ -TEST_CASE("SimplePaddingLayerTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta; - Padding module(1, 2, 3, 4); - module.InputDimensions() = std::vector({ 2, 5 }); - module.ComputeOutputDimensions(); - - // Test the Forward function. - input = arma::randu(10, 1); - size_t totalOutputDimensions = module.OutputDimensions()[0]; - for (size_t i = 1; i < module.OutputDimensions().size(); ++i) - totalOutputDimensions *= module.OutputDimensions()[i]; - output.set_size(totalOutputDimensions, input.n_cols); - output.randu(); - module.Forward(input, output); - REQUIRE(arma::accu(input) == Approx(arma::accu(output))); - REQUIRE(output.n_rows == (9 * 8)); // 2x5 --> 9x8 - - // Test the Backward function. - delta.set_size(input.n_rows, input.n_cols); - module.Backward(input, output, delta); - CheckMatrices(delta, input); - - // Test forward function for multiple filters. - // Here it's 3 filters with height = 224, width = 224 - // the output should be [226 * 226 * 3, 1] with 1 padding. - module = Padding(1, 1, 1, 1); - module.InputDimensions() = std::vector({ 224, 224, 3 }); - module.ComputeOutputDimensions(); - - input = arma::randu(224 * 224 * 3, 1); - totalOutputDimensions = module.OutputDimensions()[0]; - for (size_t i = 1; i < module.OutputDimensions().size(); ++i) - totalOutputDimensions *= module.OutputDimensions()[i]; - output.set_size(totalOutputDimensions, input.n_cols); - output.randu(); - module.Forward(input, output); - REQUIRE(arma::accu(input) == Approx(arma::accu(output))); - REQUIRE(output.n_rows == (226 * 226 * 3)); - REQUIRE(output.n_cols == 1); - - // Test forward function for multiple batches with multiple filters. - // Here it's 3 filters with height = 244, width = 244 - // the output should be [246 * 246 * 3, 3] with 1 padding. - module.InputDimensions() = std::vector({ 244, 244, 3 }); - module.ComputeOutputDimensions(); - totalOutputDimensions = module.OutputDimensions()[0]; - for (size_t i = 1; i < module.OutputDimensions().size(); ++i) - totalOutputDimensions *= module.OutputDimensions()[i]; - - input = arma::randu(244 * 244 * 3, 3); - output.set_size(totalOutputDimensions, input.n_cols); - output.randu(); - module.Forward(input, output); - REQUIRE(output.n_rows == (246 * 246 * 3)); - REQUIRE(output.n_cols == 3); - REQUIRE(arma::accu(input) == Approx(arma::accu(output))); -} - -/** - * Jacobian linear no bias module test. - */ -TEST_CASE("JacobianLinearNoBiasLayerTest", "[ANNLayerTest]") -{ - for (size_t i = 0; i < 5; ++i) - { - const size_t inputElements = math::RandInt(2, 1000); - const size_t outputElements = math::RandInt(2, 1000); - - arma::mat input; - input.set_size(inputElements, 1); - - LinearNoBias module(outputElements); - arma::mat weights(inputElements * outputElements, 1); - module.InputDimensions() = std::vector({ inputElements }); - module.ComputeOutputDimensions(); - module.SetWeights(weights.memptr()); - - module.Parameters().randu(); - - double error = JacobianTest(module, input); - REQUIRE(error <= 1e-5); - } -} - -/** - * LinearNoBias layer numerical gradient test. - */ -TEST_CASE("GradientLinearNoBiasLayerTest", "[ANNLayerTest]") -{ - // LinearNoBias function gradient instantiation. - struct GradientFunction - { - GradientFunction() : - input(arma::randu(10, 1)), - target(arma::mat("0")) - { - model = new FFN(); - model->ResetData(input, target); - model->Add(10); - model->Add(2); - model->Add(); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - REQUIRE(CheckGradient(function) <= 1e-4); -} - -/** - * Simple LogSoftMax module test. - */ -TEST_CASE("SimpleLogSoftmaxLayerTest", "[ANNLayerTest]") -{ - arma::mat output, input, error, delta; - LogSoftMax module; - - // Test the Forward function. - input = arma::mat("0.5; 0.5"); - module.Forward(input, output); - REQUIRE(arma::accu(arma::abs(arma::mat("-0.6931; -0.6931") - output)) == - Approx(0.0).margin(1e-3)); - - // Test the Backward function. - error = arma::zeros(input.n_rows, input.n_cols); - // Assume LogSoftmax layer is always associated with NLL output layer. - error(1, 0) = -1; - module.Backward(input, error, delta); - REQUIRE(arma::accu(arma::abs(arma::mat("1.6487; 0.6487") - delta)) == - Approx(0.0).margin(1e-3)); -} - -/** - * Test that the functions that can modify and access the parameters of the - * Convolution layer work. - */ -TEST_CASE("ConvolutionLayerParametersTest", "[ANNLayerTest]") -{ - // Parameter order: outSize, kW, kH, dW, dH, padW, padH, paddingType. - Convolution layer1(2, 3, 4, 5, 6, std::tuple(7, 8), - std::tuple(9, 10), "none"); - Convolution layer2(3, 4, 5, 6, 7, std::tuple(8, 9), - std::tuple(10, 11), "none"); - - // Make sure we can get the parameters successfully. - REQUIRE(layer1.KernelWidth() == 3); - REQUIRE(layer1.KernelHeight() == 4); - REQUIRE(layer1.StrideWidth() == 5); - REQUIRE(layer1.StrideHeight() == 6); - REQUIRE(layer1.PadWLeft() == 7); - REQUIRE(layer1.PadWRight() == 8); - REQUIRE(layer1.PadHTop() == 9); - REQUIRE(layer1.PadHBottom() == 10); - - // Now modify the parameters to match the second layer. - layer1.KernelWidth() = 4; - layer1.KernelHeight() = 5; - layer1.StrideWidth() = 6; - layer1.StrideHeight() = 7; - layer1.PadWLeft() = 8; - layer1.PadWRight() = 9; - layer1.PadHTop() = 10; - layer1.PadHBottom() = 11; - - // Now ensure all results are the same. - REQUIRE(layer1.KernelWidth() == layer2.KernelWidth()); - REQUIRE(layer1.KernelHeight() == layer2.KernelHeight()); - REQUIRE(layer1.StrideWidth() == layer2.StrideWidth()); - REQUIRE(layer1.StrideHeight() == layer2.StrideHeight()); - REQUIRE(layer1.PadWLeft() == layer2.PadWLeft()); - REQUIRE(layer1.PadWRight() == layer2.PadWRight()); - REQUIRE(layer1.PadHTop() == layer2.PadHTop()); - REQUIRE(layer1.PadHBottom() == layer2.PadHBottom()); -} - -/** - * Test that the padding options are working correctly in Convolution layer. - */ -TEST_CASE("ConvolutionLayerPaddingTest", "[ANNLayerTest]") -{ - arma::mat output, input, delta; - - // Check valid padding option. - Convolution module1(1, 3, 3, 1, 1, std::tuple(1, 1), - std::tuple(1, 1), "valid"); - module1.InputDimensions() = std::vector({ 7, 7 }); - module1.ComputeOutputDimensions(); - arma::mat weights1(module1.WeightSize(), 1); - REQUIRE(weights1.n_elem == 10); - module1.SetWeights(weights1.memptr()); - - // Test the Forward function. - input = arma::linspace(0, 48, 49); - output.set_size(module1.OutputSize(), 1); - module1.Parameters().zeros(); - module1.Forward(input, output); - - REQUIRE(arma::accu(output) == 0); - REQUIRE(output.n_rows == 25); - REQUIRE(output.n_cols == 1); - - // Test the Backward function. - delta.set_size(arma::size(input)); - module1.Backward(input, output, delta); - - // Check same padding option. - Convolution module2(1, 3, 3, 1, 1, std::tuple(0, 0), - std::tuple(0, 0), "same"); - module2.InputDimensions() = std::vector({ 7, 7 }); - module2.ComputeOutputDimensions(); - arma::mat weights2(module2.WeightSize(), 1); - REQUIRE(weights2.n_elem == 10); - module2.SetWeights(weights2.memptr()); - - // Test the forward function. - input = arma::linspace(0, 48, 49); - output.set_size(module2.OutputSize(), 1); - module2.Parameters().zeros(); - module2.Forward(input, output); - - REQUIRE(arma::accu(output) == 0); - REQUIRE(output.n_rows == 49); - REQUIRE(output.n_cols == 1); - - // Test the backward function. - delta.set_size(arma::size(input)); - module2.Backward(input, output, delta); -} - -/** - * Convolution layer numerical gradient test. - */ -TEST_CASE("GradientConvolutionLayerTest", "[ANNLayerTest]") -{ - struct GradientFunction - { - GradientFunction() : - input(arma::linspace(0, 35, 36)), - target(arma::mat("1")) - { - model = new FFN(); - model->ResetData(input, target); - model->Add(1, 3, 3, 1, 1, std::tuple(0, 0), - std::tuple(0, 0), "same"); - model->Add(); - - model->InputDimensions() = std::vector({ 6, 6 }); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - REQUIRE(CheckGradient(function) < 1e3); -} - -/** - * Simple test for Max Pooling layer. - */ -TEST_CASE("MaxPoolingTestCase", "[ANNLayerTest]") -{ - // For rectangular input to pooling layers. - arma::mat input = arma::mat(12, 1); - arma::mat output; - input.zeros(); - input(0) = 1; - input(1) = 2; - input(2) = 3; - input(3) = input(8) = 7; - input(4) = 4; - input(5) = 5; - input(6) = input(7) = 6; - input(10) = 8; - input(11) = 9; - // Output-Size should be 2 x 2. - output.set_size(4, 1); - - // Square output. - MaxPooling module1(2, 2, 2, 1); - module1.InputDimensions() = std::vector({ 4, 3 }); - module1.ComputeOutputDimensions(); - module1.Forward(input, output); - // Calculated using torch.nn.MaxPool2d(). - REQUIRE(arma::accu(output) == 28); - REQUIRE(output.n_elem == 4); - REQUIRE(output.n_cols == 1); - - // For Square input. - input = arma::mat(9, 1); - input.zeros(); - input(0) = 6; - input(1) = 3; - input(2) = 9; - input(3) = 3; - input(6) = 3; - // Output-Size should be 1 x 2. - output.set_size(2, 1); - - // Rectangular output. - MaxPooling module2(3, 2, 3, 1); - module2.InputDimensions() = std::vector({ 3, 3 }); - module2.ComputeOutputDimensions(); - module2.Forward(input, output); - // Calculated using torch.nn.MaxPool2d(). - REQUIRE(arma::accu(output) == 12.0); - REQUIRE(output.n_elem == 2); - REQUIRE(output.n_cols == 1); - - // For Square input. - input = arma::mat(16, 1); - input.zeros(); - input(0) = 6; - input(1) = 3; - input(2) = 9; - input(4) = 3; - input(8) = 3; - // Output-Size should be 3 x 3. - output.set_size(9, 1); - - // Square output. - MaxPooling module3(2, 2, 1, 1); - module3.InputDimensions() = std::vector({ 4, 4 }); - module3.ComputeOutputDimensions(); - module3.Forward(input, output); - // Calculated using torch.nn.MaxPool2d(). - REQUIRE(arma::accu(output) == 30.0); - REQUIRE(output.n_elem == 9); - REQUIRE(output.n_cols == 1); - - // For Rectangular input. - input = arma::mat(6, 1); - input.zeros(); - input(0) = 1; - input(1) = 1; - input(3) = 1; - // Output-Size should be 2 x 2. - output.set_size(4, 1); - - // Square output. - MaxPooling module4(2, 1, 1, 1); - module4.InputDimensions() = std::vector({ 3, 2 }); - module4.ComputeOutputDimensions(); - module4.Forward(input, output); - // Calculated using torch.nn.MaxPool2d(). - REQUIRE(arma::accu(output) == 3); - REQUIRE(output.n_elem == 4); - REQUIRE(output.n_cols == 1); -} - -TEST_CASE("ConvolutionLayerTestCase", "[ANNLayerTest]") -{ - arma::mat input, output; - - // The input test matrix is of the form 3 x 2 x 4 x 1 where - // number of images are 3 and number of feature maps are 2. - input = { { 1, 446, 42 }, - { 2, 16, 63 }, - { 3, 13, 63 }, - { 4, 21, 21 }, - { 1, 13, 11 }, - { 32, 45, 42 }, - { 22, 16 , 63 }, - { 32, 13 , 42 } }; - - Convolution layer(4, 1, 1, 1, 1, 0, 0); - layer.InputDimensions() = std::vector({ 4, 1, 2 }); - layer.ComputeOutputDimensions(); - arma::mat layerWeights(layer.WeightSize(), 1); - layer.SetWeights(layerWeights.memptr()); - output.set_size(layer.OutputSize(), 3); - - // Set weights to 1.0 and bias to 0.0. - layer.Weight().fill(1.0); - layer.Bias().zeros(); - layer.Forward(input, output); - - // Value calculated using torch.nn.Conv2d(). - REQUIRE(arma::accu(output) == 4108); - - // Set bias to one. - layer.Bias().fill(1.0); - layer.Forward(input, output); - - // Value calculated using torch.nn.Conv2d(). - REQUIRE(arma::accu(output) == 4156); -} - -/** - * Convolution module weight initialization test. - */ -TEST_CASE("ConvolutionLayerWeightInitializationTest", "[ANNLayerTest]") -{ - size_t inSize = 2, outSize = 3; - size_t kernelWidth = 4, kernelHeight = 5; - Convolution module = Convolution(outSize, - kernelWidth, kernelHeight, 6, 7, std::tuple(8, 9), - std::tuple(10, 11), "none"); - module.InputDimensions() = std::vector({ 12, 13, 2 }); - module.ComputeOutputDimensions(); - arma::mat weights(module.WeightSize(), 1); - module.SetWeights(weights.memptr()); - - RandomInitialization().Initialize(module.Weight()); - module.Bias().ones(); - - REQUIRE(std::equal(module.Weight().begin(), - module.Weight().end(), module.Parameters().begin())); - - REQUIRE(std::equal(module.Bias().begin(), - module.Bias().end(), module.Parameters().end() - outSize)); - - REQUIRE(module.Weight().n_rows == kernelWidth); - REQUIRE(module.Weight().n_cols == kernelHeight); - REQUIRE(module.Weight().n_slices == outSize * inSize); - REQUIRE(module.Bias().n_rows == outSize); - REQUIRE(module.Bias().n_cols == 1); - REQUIRE(module.Parameters().n_rows - == (outSize * inSize * kernelWidth * kernelHeight) + outSize); -} diff --git a/src/mlpack/tests/ann/ann_test_tools.hpp b/src/mlpack/tests/ann/ann_test_tools.hpp index d9f485484e..b20af96c2d 100644 --- a/src/mlpack/tests/ann/ann_test_tools.hpp +++ b/src/mlpack/tests/ann/ann_test_tools.hpp @@ -13,6 +13,8 @@ #define MLPACK_TESTS_ANN_TEST_TOOLS_HPP #include +#include +#include using namespace mlpack; using namespace mlpack::ann; diff --git a/src/mlpack/tests/ann/layer/alpha_dropout.cpp b/src/mlpack/tests/ann/layer/alpha_dropout.cpp new file mode 100644 index 0000000000..ad8759c5e9 --- /dev/null +++ b/src/mlpack/tests/ann/layer/alpha_dropout.cpp @@ -0,0 +1,116 @@ +/** + * @file tests/ann_layer_test.cpp + * @author Marcus Edel + * @author Praveen Ch + * + * Tests the ann layer modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include + +#include "../../test_catch_tools.hpp" +#include "../../catch.hpp" +#include "../../serialization.hpp" +#include "../ann_test_tools.hpp" + +using namespace mlpack; +using namespace mlpack::ann; + +/* + * Perform test to check whether mean and variance remain nearly same + * after AlphaDropout. + */ +TEST_CASE("SimpleAlphaDropoutLayerTest", "[ANNLayerTest]") +{ + // Initialize the probability of setting a value to alphaDash. + const double p = 0.2; + + // Initialize the input parameter having a mean nearabout 0 + // and variance nearabout 1. + arma::mat input = arma::randn(1000, 1); + + AlphaDropout module(p); + module.Training() = true; + + // Test the Forward function when training phase. + arma::mat output(arma::size(input)); + module.Forward(input, output); + // Check whether mean remains nearly same. + REQUIRE(arma::as_scalar(arma::abs(arma::mean(input) - arma::mean(output))) <= + 0.1); + + // Check whether variance remains nearly same. + REQUIRE(arma::as_scalar(arma::abs(arma::var(input) - arma::var(output))) <= + 0.1); + + // Test the Backward function when training phase. + arma::mat delta; + module.Backward(input, input, delta); + REQUIRE(arma::as_scalar(arma::abs(arma::mean(delta) - 0)) <= 0.05); + + // Test the Forward function when testing phase. + module.Training() = false; + module.Forward(input, output); + REQUIRE(arma::accu(input) == arma::accu(output)); +} + +/** + * Perform AlphaDropout x times using ones as input, sum the number of ones + * and validate that the layer is producing approximately the correct number + * of ones. + */ +TEST_CASE("AlphaDropoutProbabilityTest", "[ANNLayerTest]") +{ + arma::mat input = arma::ones(1500, 1); + const size_t iterations = 10; + + double probability[5] = { 0.1, 0.3, 0.4, 0.7, 0.8 }; + for (size_t trial = 0; trial < 5; ++trial) + { + double nonzeroCount = 0; + for (size_t i = 0; i < iterations; ++i) + { + AlphaDropout module(probability[trial]); + module.Training() = true; + + arma::mat output(arma::size(input)); + module.Forward(input, output); + + // Return a column vector containing the indices of elements of X + // that are not alphaDash, we just need the number of + // nonAlphaDash values. + arma::uvec nonAlphaDash = arma::find(module.Mask()); + nonzeroCount += nonAlphaDash.n_elem; + } + + const double expected = input.n_elem * (1-probability[trial]) * iterations; + + const double error = fabs(nonzeroCount - expected) / expected; + + REQUIRE(error <= 0.15); + } +} + +/** + * Perform AlphaDropout with probability 1 - p where p = 0, + * means no AlphaDropout. + */ +TEST_CASE("NoAlphaDropoutTest", "[ANNLayerTest]") +{ + arma::mat input = arma::ones(1500, 1); + AlphaDropout module(0); + module.Training() = false; + + arma::mat output; + module.Forward(input, output); + + REQUIRE(arma::accu(output) == arma::accu(input)); +} + diff --git a/src/mlpack/tests/ann/layer/convolution.cpp b/src/mlpack/tests/ann/layer/convolution.cpp new file mode 100644 index 0000000000..2bdfa4ee7d --- /dev/null +++ b/src/mlpack/tests/ann/layer/convolution.cpp @@ -0,0 +1,235 @@ +/** + * @file tests/ann_layer_test.cpp + * @author Marcus Edel + * @author Praveen Ch + * + * Tests the ann layer modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include +#include +#include + +#include "../../test_catch_tools.hpp" +#include "../../catch.hpp" +#include "../../serialization.hpp" +#include "../ann_test_tools.hpp" + +using namespace mlpack; +using namespace mlpack::ann; + +/** + * Test that the functions that can modify and access the parameters of the + * Convolution layer work. + */ +TEST_CASE("ConvolutionLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order: outSize, kW, kH, dW, dH, padW, padH, paddingType. + Convolution layer1(2, 3, 4, 5, 6, std::tuple(7, 8), + std::tuple(9, 10), "none"); + Convolution layer2(3, 4, 5, 6, 7, std::tuple(8, 9), + std::tuple(10, 11), "none"); + + // Make sure we can get the parameters successfully. + REQUIRE(layer1.KernelWidth() == 3); + REQUIRE(layer1.KernelHeight() == 4); + REQUIRE(layer1.StrideWidth() == 5); + REQUIRE(layer1.StrideHeight() == 6); + REQUIRE(layer1.PadWLeft() == 7); + REQUIRE(layer1.PadWRight() == 8); + REQUIRE(layer1.PadHTop() == 9); + REQUIRE(layer1.PadHBottom() == 10); + + // Now modify the parameters to match the second layer. + layer1.KernelWidth() = 4; + layer1.KernelHeight() = 5; + layer1.StrideWidth() = 6; + layer1.StrideHeight() = 7; + layer1.PadWLeft() = 8; + layer1.PadWRight() = 9; + layer1.PadHTop() = 10; + layer1.PadHBottom() = 11; + + // Now ensure all results are the same. + REQUIRE(layer1.KernelWidth() == layer2.KernelWidth()); + REQUIRE(layer1.KernelHeight() == layer2.KernelHeight()); + REQUIRE(layer1.StrideWidth() == layer2.StrideWidth()); + REQUIRE(layer1.StrideHeight() == layer2.StrideHeight()); + REQUIRE(layer1.PadWLeft() == layer2.PadWLeft()); + REQUIRE(layer1.PadWRight() == layer2.PadWRight()); + REQUIRE(layer1.PadHTop() == layer2.PadHTop()); + REQUIRE(layer1.PadHBottom() == layer2.PadHBottom()); +} + +/** + * Test that the padding options are working correctly in Convolution layer. + */ +TEST_CASE("ConvolutionLayerPaddingTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + + // Check valid padding option. + Convolution module1(1, 3, 3, 1, 1, std::tuple(1, 1), + std::tuple(1, 1), "valid"); + module1.InputDimensions() = std::vector({ 7, 7 }); + module1.ComputeOutputDimensions(); + arma::mat weights1(module1.WeightSize(), 1); + REQUIRE(weights1.n_elem == 10); + module1.SetWeights(weights1.memptr()); + + // Test the Forward function. + input = arma::linspace(0, 48, 49); + output.set_size(module1.OutputSize(), 1); + module1.Parameters().zeros(); + module1.Forward(input, output); + + REQUIRE(arma::accu(output) == 0); + REQUIRE(output.n_rows == 25); + REQUIRE(output.n_cols == 1); + + // Test the Backward function. + delta.set_size(arma::size(input)); + module1.Backward(input, output, delta); + + // Check same padding option. + Convolution module2(1, 3, 3, 1, 1, std::tuple(0, 0), + std::tuple(0, 0), "same"); + module2.InputDimensions() = std::vector({ 7, 7 }); + module2.ComputeOutputDimensions(); + arma::mat weights2(module2.WeightSize(), 1); + REQUIRE(weights2.n_elem == 10); + module2.SetWeights(weights2.memptr()); + + // Test the forward function. + input = arma::linspace(0, 48, 49); + output.set_size(module2.OutputSize(), 1); + module2.Parameters().zeros(); + module2.Forward(input, output); + + REQUIRE(arma::accu(output) == 0); + REQUIRE(output.n_rows == 49); + REQUIRE(output.n_cols == 1); + + // Test the backward function. + delta.set_size(arma::size(input)); + module2.Backward(input, output, delta); +} + +/** + * Convolution layer numerical gradient test. + */ +TEST_CASE("GradientConvolutionLayerTest", "[ANNLayerTest]") +{ + struct GradientFunction + { + GradientFunction() : + input(arma::linspace(0, 35, 36)), + target(arma::mat("1")) + { + model = new FFN(); + model->ResetData(input, target); + model->Add(1, 3, 3, 1, 1, std::tuple(0, 0), + std::tuple(0, 0), "same"); + model->Add(); + + model->InputDimensions() = std::vector({ 6, 6 }); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + REQUIRE(CheckGradient(function) < 1e3); +} + +TEST_CASE("ConvolutionLayerTestCase", "[ANNLayerTest]") +{ + arma::mat input, output; + + // The input test matrix is of the form 3 x 2 x 4 x 1 where + // number of images are 3 and number of feature maps are 2. + input = { { 1, 446, 42 }, + { 2, 16, 63 }, + { 3, 13, 63 }, + { 4, 21, 21 }, + { 1, 13, 11 }, + { 32, 45, 42 }, + { 22, 16 , 63 }, + { 32, 13 , 42 } }; + + Convolution layer(4, 1, 1, 1, 1, 0, 0); + layer.InputDimensions() = std::vector({ 4, 1, 2 }); + layer.ComputeOutputDimensions(); + arma::mat layerWeights(layer.WeightSize(), 1); + layer.SetWeights(layerWeights.memptr()); + output.set_size(layer.OutputSize(), 3); + + // Set weights to 1.0 and bias to 0.0. + layer.Weight().fill(1.0); + layer.Bias().zeros(); + layer.Forward(input, output); + + // Value calculated using torch.nn.Conv2d(). + REQUIRE(arma::accu(output) == 4108); + + // Set bias to one. + layer.Bias().fill(1.0); + layer.Forward(input, output); + + // Value calculated using torch.nn.Conv2d(). + REQUIRE(arma::accu(output) == 4156); +} + +/** + * Convolution module weight initialization test. + */ +TEST_CASE("ConvolutionLayerWeightInitializationTest", "[ANNLayerTest]") +{ + size_t inSize = 2, outSize = 3; + size_t kernelWidth = 4, kernelHeight = 5; + Convolution module = Convolution(outSize, + kernelWidth, kernelHeight, 6, 7, std::tuple(8, 9), + std::tuple(10, 11), "none"); + module.InputDimensions() = std::vector({ 12, 13, 2 }); + module.ComputeOutputDimensions(); + arma::mat weights(module.WeightSize(), 1); + module.SetWeights(weights.memptr()); + + RandomInitialization().Initialize(module.Weight()); + module.Bias().ones(); + + REQUIRE(std::equal(module.Weight().begin(), + module.Weight().end(), module.Parameters().begin())); + + REQUIRE(std::equal(module.Bias().begin(), + module.Bias().end(), module.Parameters().end() - outSize)); + + REQUIRE(module.Weight().n_rows == kernelWidth); + REQUIRE(module.Weight().n_cols == kernelHeight); + REQUIRE(module.Weight().n_slices == outSize * inSize); + REQUIRE(module.Bias().n_rows == outSize); + REQUIRE(module.Bias().n_cols == 1); + REQUIRE(module.Parameters().n_rows + == (outSize * inSize * kernelWidth * kernelHeight) + outSize); +} diff --git a/src/mlpack/tests/ann/layer/dropout.cpp b/src/mlpack/tests/ann/layer/dropout.cpp new file mode 100644 index 0000000000..0b7b45314f --- /dev/null +++ b/src/mlpack/tests/ann/layer/dropout.cpp @@ -0,0 +1,105 @@ +/** + * @file tests/ann_layer_test.cpp + * @author Marcus Edel + * @author Praveen Ch + * + * Tests the ann layer modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include + +#include "../../test_catch_tools.hpp" +#include "../../catch.hpp" +#include "../ann_test_tools.hpp" + +using namespace mlpack; +using namespace mlpack::ann; + +/** + * Simple dropout module test. + */ +TEST_CASE("SimpleDropoutLayerTest", "[ANNLayerTest]") +{ + // Initialize the probability of setting a value to zero. + const double p = 0.2; + + // Initialize the input parameter. + arma::mat input(1000, 1); + input.fill(1 - p); + + Dropout module(p); + module.Training() = true; + + // Test the Forward function. + arma::mat output; + module.Forward(input, output); + REQUIRE(arma::as_scalar(arma::abs(arma::mean(output) - (1 - p))) <= 0.05); + + // Test the Backward function. + arma::mat delta; + module.Backward(input, input, delta); + REQUIRE(arma::as_scalar(arma::abs(arma::mean(delta) - (1 - p))) <= 0.05); + + // Test the Forward function. + module.Training() = false; + module.Forward(input, output); + REQUIRE(arma::accu(input) == arma::accu(output)); +} + +/** + * Perform dropout x times using ones as input, sum the number of ones and + * validate that the layer is producing approximately the correct number of + * ones. + */ +TEST_CASE("DropoutProbabilityTest", "[ANNLayerTest]") +{ + arma::mat input = arma::ones(1500, 1); + const size_t iterations = 10; + + double probability[5] = { 0.1, 0.3, 0.4, 0.7, 0.8 }; + for (size_t trial = 0; trial < 5; ++trial) + { + double nonzeroCount = 0; + for (size_t i = 0; i < iterations; ++i) + { + Dropout module(probability[trial]); + module.Training() = true; + + arma::mat output; + module.Forward(input, output); + + // Return a column vector containing the indices of elements of X that + // are non-zero, we just need the number of non-zero values. + arma::uvec nonzero = arma::find(output); + nonzeroCount += nonzero.n_elem; + } + const double expected = input.n_elem * (1 - probability[trial]) * + iterations; + const double error = fabs(nonzeroCount - expected) / expected; + + REQUIRE(error <= 0.15); + } +} + +/* + * Perform dropout with probability 1 - p where p = 0, means no dropout. + */ +TEST_CASE("NoDropoutTest", "[ANNLayerTest]") +{ + arma::mat input = arma::ones(1500, 1); + Dropout module(0); + module.Training() = true; + + arma::mat output; + module.Forward(input, output); + + REQUIRE(arma::accu(output) == arma::accu(input)); +} + diff --git a/src/mlpack/tests/ann/layer/linear3d.cpp b/src/mlpack/tests/ann/layer/linear3d.cpp new file mode 100644 index 0000000000..92843e8d81 --- /dev/null +++ b/src/mlpack/tests/ann/layer/linear3d.cpp @@ -0,0 +1,142 @@ +/** + * @file tests/ann_layer_test.cpp + * @author Marcus Edel + * @author Praveen Ch + * + * Tests the ann layer modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include +#include +#include +#include + +#include "../../test_catch_tools.hpp" +#include "../../catch.hpp" +#include "../../serialization.hpp" +#include "../ann_test_tools.hpp" + +using namespace mlpack; +using namespace mlpack::ann; + +/** + * Simple Linear3D layer test. + */ +TEST_CASE("SimpleLinear3DLayerTest", "[ANNLayerTest]") +{ + const size_t inSize = 4; + const size_t outSize = 1; + const size_t nPoints = 2; + const size_t batchSize = 1; + arma::mat input, output, delta; + + // Create a Linear3D layer outside of a network, and then set its memory. + Linear3D module(outSize); + module.InputDimensions() = std::vector({ 4, 2 }); + module.ComputeOutputDimensions(); + arma::mat weights(module.WeightSize(), 1); + module.SetWeights(weights.memptr()); + + module.Parameters().randu(); + + // Test the Forward function. + input = arma::zeros(inSize * nPoints, batchSize); + output.set_size(outSize * nPoints, batchSize); + module.Forward(input, output); + REQUIRE(arma::accu(module.Bias()) + == Approx(arma::accu(output) / (nPoints * batchSize)).epsilon(1e-3)); + + // Test the Backward function. + delta.set_size(input.n_rows, input.n_cols); + output.zeros(); + module.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 0); +} + +/** + * Jacobian Linear3D module test. + */ +TEST_CASE("JacobianLinear3DLayerTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t inSize = math::RandInt(2, 10); + const size_t outSize = math::RandInt(2, 10); + const size_t nPoints = math::RandInt(2, 10); + const size_t batchSize = 1; + + arma::mat input; + input.set_size(inSize * nPoints, batchSize); + + // Create a Linear3D layer outside a network and initialize its memory. + Linear3D module(outSize); + module.InputDimensions() = std::vector({ inSize, nPoints }); + module.ComputeOutputDimensions(); + arma::mat weights(module.WeightSize(), 1); + module.SetWeights(weights.memptr()); + + module.Parameters().randu(); + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} + +/** + * Simple Gradient test for Linear3D layer. + */ +TEST_CASE("GradientLinear3DLayerTest", "[ANNLayerTest]") +{ + // Linear function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + inSize(4), + outSize(1), + nPoints(2), + batchSize(4) + { + input = arma::randu(inSize * nPoints, batchSize); + target = arma::zeros(outSize * nPoints, batchSize); + target(0, 0) = 1; + target(0, 3) = 1; + target(1, 1) = 1; + target(1, 2) = 1; + + model = new FFN(); + model->ResetData(input, target); + model->Add(outSize); + model->InputDimensions() = std::vector{ 4, 2 }; + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + const size_t inSize; + const size_t outSize; + const size_t nPoints; + const size_t batchSize; + } function; + + REQUIRE(CheckGradient(function) <= 1e-7); +} diff --git a/src/mlpack/tests/ann/layer/linear_no_bias.cpp b/src/mlpack/tests/ann/layer/linear_no_bias.cpp new file mode 100644 index 0000000000..c95e289c0d --- /dev/null +++ b/src/mlpack/tests/ann/layer/linear_no_bias.cpp @@ -0,0 +1,116 @@ +/** + * @file tests/ann_layer_test.cpp + * @author Marcus Edel + * @author Praveen Ch + * + * Tests the ann layer modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include +#include +#include + +#include "../../test_catch_tools.hpp" +#include "../../catch.hpp" +#include "../../serialization.hpp" +#include "../ann_test_tools.hpp" + +using namespace mlpack; +using namespace mlpack::ann; + +/** + * Simple linear no bias module test. + */ +TEST_CASE("SimpleLinearNoBiasLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + LinearNoBias module(10); + arma::mat weights(10 * 10, 1); + module.InputDimensions() = std::vector({ 10 }); + module.ComputeOutputDimensions(); + module.SetWeights(weights.memptr()); + + module.Parameters().randu(); + + // Test the Forward function. + input = arma::zeros(10, 1); + module.Forward(input, output); + REQUIRE(0 == arma::accu(output)); + + // Test the Backward function. + module.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 0); +} + +/** + * Jacobian linear no bias module test. + */ +TEST_CASE("JacobianLinearNoBiasLayerTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t inputElements = math::RandInt(2, 1000); + const size_t outputElements = math::RandInt(2, 1000); + + arma::mat input; + input.set_size(inputElements, 1); + + LinearNoBias module(outputElements); + arma::mat weights(inputElements * outputElements, 1); + module.InputDimensions() = std::vector({ inputElements }); + module.ComputeOutputDimensions(); + module.SetWeights(weights.memptr()); + + module.Parameters().randu(); + + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} + +/** + * LinearNoBias layer numerical gradient test. + */ +TEST_CASE("GradientLinearNoBiasLayerTest", "[ANNLayerTest]") +{ + // LinearNoBias function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(10, 1)), + target(arma::mat("0")) + { + model = new FFN(); + model->ResetData(input, target); + model->Add(10); + model->Add(2); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + REQUIRE(CheckGradient(function) <= 1e-4); +} diff --git a/src/mlpack/tests/ann/layer/log_softmax.cpp b/src/mlpack/tests/ann/layer/log_softmax.cpp new file mode 100644 index 0000000000..720dc8e16c --- /dev/null +++ b/src/mlpack/tests/ann/layer/log_softmax.cpp @@ -0,0 +1,49 @@ +/** + * @file tests/ann_layer_test.cpp + * @author Marcus Edel + * @author Praveen Ch + * + * Tests the ann layer modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include +#include +#include + +#include "../../test_catch_tools.hpp" +#include "../../catch.hpp" +#include "../../serialization.hpp" +#include "../ann_test_tools.hpp" + +using namespace mlpack; +using namespace mlpack::ann; + +/** + * Simple LogSoftMax module test. + */ +TEST_CASE("SimpleLogSoftmaxLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, error, delta; + LogSoftMax module; + + // Test the Forward function. + input = arma::mat("0.5; 0.5"); + module.Forward(input, output); + REQUIRE(arma::accu(arma::abs(arma::mat("-0.6931; -0.6931") - output)) == + Approx(0.0).margin(1e-3)); + + // Test the Backward function. + error = arma::zeros(input.n_rows, input.n_cols); + // Assume LogSoftmax layer is always associated with NLL output layer. + error(1, 0) = -1; + module.Backward(input, error, delta); + REQUIRE(arma::accu(arma::abs(arma::mat("1.6487; 0.6487") - delta)) == + Approx(0.0).margin(1e-3)); +} diff --git a/src/mlpack/tests/ann/layer/max_pooling.cpp b/src/mlpack/tests/ann/layer/max_pooling.cpp new file mode 100644 index 0000000000..20508fddcb --- /dev/null +++ b/src/mlpack/tests/ann/layer/max_pooling.cpp @@ -0,0 +1,118 @@ +/** + * @file tests/ann_layer_test.cpp + * @author Marcus Edel + * @author Praveen Ch + * + * Tests the ann layer modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include +#include + +#include "../../test_catch_tools.hpp" +#include "../../catch.hpp" +#include "../../serialization.hpp" +#include "../ann_test_tools.hpp" + +using namespace mlpack; +using namespace mlpack::ann; + +/** + * Simple test for Max Pooling layer. + */ +TEST_CASE("MaxPoolingTestCase", "[ANNLayerTest]") +{ + // For rectangular input to pooling layers. + arma::mat input = arma::mat(12, 1); + arma::mat output; + input.zeros(); + input(0) = 1; + input(1) = 2; + input(2) = 3; + input(3) = input(8) = 7; + input(4) = 4; + input(5) = 5; + input(6) = input(7) = 6; + input(10) = 8; + input(11) = 9; + // Output-Size should be 2 x 2. + output.set_size(4, 1); + + // Square output. + MaxPooling module1(2, 2, 2, 1); + module1.InputDimensions() = std::vector({ 4, 3 }); + module1.ComputeOutputDimensions(); + module1.Forward(input, output); + // Calculated using torch.nn.MaxPool2d(). + REQUIRE(arma::accu(output) == 28); + REQUIRE(output.n_elem == 4); + REQUIRE(output.n_cols == 1); + + // For Square input. + input = arma::mat(9, 1); + input.zeros(); + input(0) = 6; + input(1) = 3; + input(2) = 9; + input(3) = 3; + input(6) = 3; + // Output-Size should be 1 x 2. + output.set_size(2, 1); + + // Rectangular output. + MaxPooling module2(3, 2, 3, 1); + module2.InputDimensions() = std::vector({ 3, 3 }); + module2.ComputeOutputDimensions(); + module2.Forward(input, output); + // Calculated using torch.nn.MaxPool2d(). + REQUIRE(arma::accu(output) == 12.0); + REQUIRE(output.n_elem == 2); + REQUIRE(output.n_cols == 1); + + // For Square input. + input = arma::mat(16, 1); + input.zeros(); + input(0) = 6; + input(1) = 3; + input(2) = 9; + input(4) = 3; + input(8) = 3; + // Output-Size should be 3 x 3. + output.set_size(9, 1); + + // Square output. + MaxPooling module3(2, 2, 1, 1); + module3.InputDimensions() = std::vector({ 4, 4 }); + module3.ComputeOutputDimensions(); + module3.Forward(input, output); + // Calculated using torch.nn.MaxPool2d(). + REQUIRE(arma::accu(output) == 30.0); + REQUIRE(output.n_elem == 9); + REQUIRE(output.n_cols == 1); + + // For Rectangular input. + input = arma::mat(6, 1); + input.zeros(); + input(0) = 1; + input(1) = 1; + input(3) = 1; + // Output-Size should be 2 x 2. + output.set_size(4, 1); + + // Square output. + MaxPooling module4(2, 1, 1, 1); + module4.InputDimensions() = std::vector({ 3, 2 }); + module4.ComputeOutputDimensions(); + module4.Forward(input, output); + // Calculated using torch.nn.MaxPool2d(). + REQUIRE(arma::accu(output) == 3); + REQUIRE(output.n_elem == 4); + REQUIRE(output.n_cols == 1); +} diff --git a/src/mlpack/tests/ann/layer/padding.cpp b/src/mlpack/tests/ann/layer/padding.cpp new file mode 100644 index 0000000000..a85b0ec517 --- /dev/null +++ b/src/mlpack/tests/ann/layer/padding.cpp @@ -0,0 +1,87 @@ +/** + * @file tests/ann_layer_test.cpp + * @author Marcus Edel + * @author Praveen Ch + * + * Tests the ann layer modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include +#include + +#include "../../test_catch_tools.hpp" +#include "../../catch.hpp" +#include "../../serialization.hpp" +#include "../ann_test_tools.hpp" + +using namespace mlpack; +using namespace mlpack::ann; + +/** + * Simple padding layer test. + */ +TEST_CASE("SimplePaddingLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + Padding module(1, 2, 3, 4); + module.InputDimensions() = std::vector({ 2, 5 }); + module.ComputeOutputDimensions(); + + // Test the Forward function. + input = arma::randu(10, 1); + size_t totalOutputDimensions = module.OutputDimensions()[0]; + for (size_t i = 1; i < module.OutputDimensions().size(); ++i) + totalOutputDimensions *= module.OutputDimensions()[i]; + output.set_size(totalOutputDimensions, input.n_cols); + output.randu(); + module.Forward(input, output); + REQUIRE(arma::accu(input) == Approx(arma::accu(output))); + REQUIRE(output.n_rows == (9 * 8)); // 2x5 --> 9x8 + + // Test the Backward function. + delta.set_size(input.n_rows, input.n_cols); + module.Backward(input, output, delta); + CheckMatrices(delta, input); + + // Test forward function for multiple filters. + // Here it's 3 filters with height = 224, width = 224 + // the output should be [226 * 226 * 3, 1] with 1 padding. + module = Padding(1, 1, 1, 1); + module.InputDimensions() = std::vector({ 224, 224, 3 }); + module.ComputeOutputDimensions(); + + input = arma::randu(224 * 224 * 3, 1); + totalOutputDimensions = module.OutputDimensions()[0]; + for (size_t i = 1; i < module.OutputDimensions().size(); ++i) + totalOutputDimensions *= module.OutputDimensions()[i]; + output.set_size(totalOutputDimensions, input.n_cols); + output.randu(); + module.Forward(input, output); + REQUIRE(arma::accu(input) == Approx(arma::accu(output))); + REQUIRE(output.n_rows == (226 * 226 * 3)); + REQUIRE(output.n_cols == 1); + + // Test forward function for multiple batches with multiple filters. + // Here it's 3 filters with height = 244, width = 244 + // the output should be [246 * 246 * 3, 3] with 1 padding. + module.InputDimensions() = std::vector({ 244, 244, 3 }); + module.ComputeOutputDimensions(); + totalOutputDimensions = module.OutputDimensions()[0]; + for (size_t i = 1; i < module.OutputDimensions().size(); ++i) + totalOutputDimensions *= module.OutputDimensions()[i]; + + input = arma::randu(244 * 244 * 3, 3); + output.set_size(totalOutputDimensions, input.n_cols); + output.randu(); + module.Forward(input, output); + REQUIRE(output.n_rows == (246 * 246 * 3)); + REQUIRE(output.n_cols == 3); + REQUIRE(arma::accu(input) == Approx(arma::accu(output))); +} diff --git a/src/mlpack/tests/ann/layer_test.cpp b/src/mlpack/tests/ann/layer_test.cpp new file mode 100644 index 0000000000..6c48539122 --- /dev/null +++ b/src/mlpack/tests/ann/layer_test.cpp @@ -0,0 +1,21 @@ +/** + * @file layer_test.cpp + * @author Ryan Curtin + * + * This file includes all tests in ann/layer/, which are split up by layer for + * organization. However, compiling each test individually results in a huge + * amount of compilation overhead; including them all here into one file reduces + * compilation time and memory usage. + * + * It's possible that this could be avoided by smart use of extern template + * instantiations. + */ + +#include "layer/alpha_dropout.cpp" +#include "layer/convolution.cpp" +#include "layer/dropout.cpp" +#include "layer/linear3d.cpp" +#include "layer/linear_no_bias.cpp" +#include "layer/log_softmax.cpp" +#include "layer/max_pooling.cpp" +#include "layer/padding.cpp" From 4b57b60a1e0bbf00f51cd729f5ff7446848dfcda Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sat, 14 May 2022 13:30:22 -0400 Subject: [PATCH 05/39] Use 0-indexed labels throughout. --- src/mlpack/tests/ann/feedforward_network_test.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/mlpack/tests/ann/feedforward_network_test.cpp b/src/mlpack/tests/ann/feedforward_network_test.cpp index 21d3a2ae3b..6d4a60efed 100644 --- a/src/mlpack/tests/ann/feedforward_network_test.cpp +++ b/src/mlpack/tests/ann/feedforward_network_test.cpp @@ -916,20 +916,20 @@ TEST_CASE("RBFNetworkTest", "[FeedForwardNetworkTest]") if (!data::Load("thyroid_train.csv", trainData)) FAIL("Cannot open thyroid_train.csv"); - arma::mat trainLabels = trainData.row(trainData.n_rows - 1); + arma::mat trainLabels = trainData.row(trainData.n_rows - 1) - 1; trainData.shed_row(trainData.n_rows - 1); arma::mat trainLabels1 = arma::zeros(3, trainData.n_cols); for (size_t i = 0; i < trainData.n_cols; i++) { - trainLabels1.col(i).row((trainLabels(i) - 1)) = 1; + trainLabels1.col(i).row(trainLabels(i)) = 1; } arma::mat testData; if (!data::Load("thyroid_test.csv", testData)) FAIL("Cannot open thyroid_test.csv"); - arma::mat testLabels = testData.row(testData.n_rows - 1); + arma::mat testLabels = testData.row(testData.n_rows - 1) - 1; testData.shed_row(testData.n_rows - 1); /* @@ -974,7 +974,6 @@ TEST_CASE("RBFNetworkTest", "[FeedForwardNetworkTest]") { labels1.col(i).row(labels(i)) = 1; } - labels += 1; arma::mat centroids1; arma::Row assignments; From 0c7ea3ea40c0eb71dbcbf53ffe7db8bcc99d24fc Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 10 Jun 2022 17:40:52 -0400 Subject: [PATCH 06/39] Refactor new tests into their own files. --- src/mlpack/tests/ann/layer/alpha_dropout.cpp | 2 +- src/mlpack/tests/ann/layer/convolution.cpp | 2 +- src/mlpack/tests/ann/layer/dropout.cpp | 2 +- src/mlpack/tests/ann/layer/linear3d.cpp | 2 +- src/mlpack/tests/ann/layer/linear_no_bias.cpp | 2 +- src/mlpack/tests/ann/layer/log_softmax.cpp | 2 +- src/mlpack/tests/ann/layer/max_pooling.cpp | 2 +- src/mlpack/tests/ann/layer/padding.cpp | 2 +- src/mlpack/tests/ann/layer_test.cpp | 4 ++++ 9 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/mlpack/tests/ann/layer/alpha_dropout.cpp b/src/mlpack/tests/ann/layer/alpha_dropout.cpp index ad8759c5e9..066b940e33 100644 --- a/src/mlpack/tests/ann/layer/alpha_dropout.cpp +++ b/src/mlpack/tests/ann/layer/alpha_dropout.cpp @@ -1,5 +1,5 @@ /** - * @file tests/ann_layer_test.cpp + * @file tests/ann/layer/alpha_dropout.cpp * @author Marcus Edel * @author Praveen Ch * diff --git a/src/mlpack/tests/ann/layer/convolution.cpp b/src/mlpack/tests/ann/layer/convolution.cpp index 2bdfa4ee7d..442935ee73 100644 --- a/src/mlpack/tests/ann/layer/convolution.cpp +++ b/src/mlpack/tests/ann/layer/convolution.cpp @@ -1,5 +1,5 @@ /** - * @file tests/ann_layer_test.cpp + * @file tests/ann/layer/convolution.cpp * @author Marcus Edel * @author Praveen Ch * diff --git a/src/mlpack/tests/ann/layer/dropout.cpp b/src/mlpack/tests/ann/layer/dropout.cpp index 0b7b45314f..3da290502f 100644 --- a/src/mlpack/tests/ann/layer/dropout.cpp +++ b/src/mlpack/tests/ann/layer/dropout.cpp @@ -1,5 +1,5 @@ /** - * @file tests/ann_layer_test.cpp + * @file tests/ann/layer/dropout.cpp * @author Marcus Edel * @author Praveen Ch * diff --git a/src/mlpack/tests/ann/layer/linear3d.cpp b/src/mlpack/tests/ann/layer/linear3d.cpp index 92843e8d81..a8f73b21ce 100644 --- a/src/mlpack/tests/ann/layer/linear3d.cpp +++ b/src/mlpack/tests/ann/layer/linear3d.cpp @@ -1,5 +1,5 @@ /** - * @file tests/ann_layer_test.cpp + * @file tests/ann/layer/linear3d.cpp * @author Marcus Edel * @author Praveen Ch * diff --git a/src/mlpack/tests/ann/layer/linear_no_bias.cpp b/src/mlpack/tests/ann/layer/linear_no_bias.cpp index c95e289c0d..5a56e37cb8 100644 --- a/src/mlpack/tests/ann/layer/linear_no_bias.cpp +++ b/src/mlpack/tests/ann/layer/linear_no_bias.cpp @@ -1,5 +1,5 @@ /** - * @file tests/ann_layer_test.cpp + * @file tests/ann/layer/linear_no_bias.cpp * @author Marcus Edel * @author Praveen Ch * diff --git a/src/mlpack/tests/ann/layer/log_softmax.cpp b/src/mlpack/tests/ann/layer/log_softmax.cpp index 720dc8e16c..2686c90eff 100644 --- a/src/mlpack/tests/ann/layer/log_softmax.cpp +++ b/src/mlpack/tests/ann/layer/log_softmax.cpp @@ -1,5 +1,5 @@ /** - * @file tests/ann_layer_test.cpp + * @file tests/ann/layer/log_softmax.cpp * @author Marcus Edel * @author Praveen Ch * diff --git a/src/mlpack/tests/ann/layer/max_pooling.cpp b/src/mlpack/tests/ann/layer/max_pooling.cpp index 20508fddcb..300181a28e 100644 --- a/src/mlpack/tests/ann/layer/max_pooling.cpp +++ b/src/mlpack/tests/ann/layer/max_pooling.cpp @@ -1,5 +1,5 @@ /** - * @file tests/ann_layer_test.cpp + * @file tests/ann/layer/max_pooling.cpp * @author Marcus Edel * @author Praveen Ch * diff --git a/src/mlpack/tests/ann/layer/padding.cpp b/src/mlpack/tests/ann/layer/padding.cpp index a85b0ec517..2a25baed50 100644 --- a/src/mlpack/tests/ann/layer/padding.cpp +++ b/src/mlpack/tests/ann/layer/padding.cpp @@ -1,5 +1,5 @@ /** - * @file tests/ann_layer_test.cpp + * @file tests/ann/layer/padding.cpp * @author Marcus Edel * @author Praveen Ch * diff --git a/src/mlpack/tests/ann/layer_test.cpp b/src/mlpack/tests/ann/layer_test.cpp index 6c48539122..b791cb472b 100644 --- a/src/mlpack/tests/ann/layer_test.cpp +++ b/src/mlpack/tests/ann/layer_test.cpp @@ -11,6 +11,8 @@ * instantiations. */ +#include "layer/adaptive_max_pooling.cpp" +#include "layer/adaptive_mean_pooling.cpp" #include "layer/alpha_dropout.cpp" #include "layer/convolution.cpp" #include "layer/dropout.cpp" @@ -18,4 +20,6 @@ #include "layer/linear_no_bias.cpp" #include "layer/log_softmax.cpp" #include "layer/max_pooling.cpp" +#include "layer/mean_pooling.cpp" #include "layer/padding.cpp" +#include "layer/softmax.cpp" From 164c105d1396666f8eb3f344289a36226f0c743d Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 10 Jun 2022 17:49:04 -0400 Subject: [PATCH 07/39] Oops, add new files. --- .../tests/ann/layer/adaptive_max_pooling.cpp | 141 ++++++++++++++++++ .../tests/ann/layer/adaptive_mean_pooling.cpp | 137 +++++++++++++++++ src/mlpack/tests/ann/layer/mean_pooling.cpp | 99 ++++++++++++ src/mlpack/tests/ann/layer/softmax.cpp | 94 ++++++++++++ 4 files changed, 471 insertions(+) create mode 100644 src/mlpack/tests/ann/layer/adaptive_max_pooling.cpp create mode 100644 src/mlpack/tests/ann/layer/adaptive_mean_pooling.cpp create mode 100644 src/mlpack/tests/ann/layer/mean_pooling.cpp create mode 100644 src/mlpack/tests/ann/layer/softmax.cpp diff --git a/src/mlpack/tests/ann/layer/adaptive_max_pooling.cpp b/src/mlpack/tests/ann/layer/adaptive_max_pooling.cpp new file mode 100644 index 0000000000..3acabad1ed --- /dev/null +++ b/src/mlpack/tests/ann/layer/adaptive_max_pooling.cpp @@ -0,0 +1,141 @@ +/** + * @file tests/ann/layer/adaptive_max_pooling.cpp + * @author Marcus Edel + * @author Praveen Ch + * + * Tests the ann layer modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../test_catch_tools.hpp" +#include "../../catch.hpp" +#include "../../serialization.hpp" +#include "../ann_test_tools.hpp" + +using namespace mlpack; +using namespace mlpack::ann; + +/** + * Simple test for Adaptive pooling for Max Pooling layer. + */ +TEST_CASE("AdaptiveMaxPoolingTestCase", "[ANNLayerTest]") +{ + // For rectangular input. + arma::mat input = arma::mat(12, 1); + arma::mat output, delta; + + input.zeros(); + input(0) = 1; + input(1) = 2; + input(2) = 3; + input(3) = input(8) = 7; + input(4) = 4; + input(5) = 5; + input(6) = input(7) = 6; + input(10) = 8; + input(11) = 9; + // Output-Size should be 2 x 2. + // Square output. + AdaptiveMaxPooling module1(2, 2); + output.set_size(4, 1); + module1.InputDimensions() = std::vector({ 4, 3 }); + module1.ComputeOutputDimensions(); + module1.Training() = true; + module1.Forward(input, output); + // Calculated using torch.nn.AdaptiveMaxPool2d(). + REQUIRE(arma::accu(output) == 28); + REQUIRE(output.n_elem == 4); + REQUIRE(output.n_cols == 1); + // Test the Backward Function. + delta.set_size(12, 1); + module1.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 28.0); + + // For Square input. + input = arma::mat(9, 1); + input.zeros(); + input(0) = 6; + input(1) = 3; + input(2) = 9; + input(3) = 3; + input(6) = 3; + // Output-Size should be 1 x 2. + // Rectangular output. + AdaptiveMaxPooling module2(2, 1); + output.set_size(2, 1); + module2.InputDimensions() = std::vector({ 3, 3 }); + module2.ComputeOutputDimensions(); + module2.Training() = true; + module2.Forward(input, output); + // Calculated using torch.nn.AdaptiveMaxPool2d(). + REQUIRE(arma::accu(output) == 15.0); + REQUIRE(output.n_elem == 2); + REQUIRE(output.n_cols == 1); + // Test the Backward Function. + delta.set_size(9, 1); + module2.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 15.0); + + // For Square input. + input = arma::mat(16, 1); + input.zeros(); + input(0) = 6; + input(1) = 3; + input(2) = 9; + input(4) = 3; + input(8) = 3; + // Output-Size should be 3 x 3. + // Square output. + AdaptiveMaxPooling module3(3, 3); + output.set_size(9, 1); + module3.InputDimensions() = std::vector({ 4, 4 }); + module3.ComputeOutputDimensions(); + module3.Training() = true; + module3.Forward(input, output); + // Calculated using torch.nn.AdaptiveMaxPool2d(). + REQUIRE(arma::accu(output) == 30.0); + REQUIRE(output.n_elem == 9); + REQUIRE(output.n_cols == 1); + // Test the Backward Function. + delta.set_size(16, 1); + module3.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 30.0); + + // For Rectangular input. + input = arma::mat(20, 1); + input.zeros(); + input(0) = 1; + input(1) = 1; + input(3) = 1; + // Output-Size should be 2 x 2. + // Square output. + AdaptiveMaxPooling module4(2, 2); + output.set_size(4, 1); + module4.InputDimensions() = std::vector({ 5, 4 }); + module4.ComputeOutputDimensions(); + module4.Training() = true; + module4.Forward(input, output); + // Calculated using torch.nn.AdaptiveMaxPool2d(). + REQUIRE(arma::accu(output) == 2); + REQUIRE(output.n_elem == 4); + REQUIRE(output.n_cols == 1); + // Test the Backward Function. + delta.set_size(20, 1); + module4.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 2.0); +} diff --git a/src/mlpack/tests/ann/layer/adaptive_mean_pooling.cpp b/src/mlpack/tests/ann/layer/adaptive_mean_pooling.cpp new file mode 100644 index 0000000000..11af81406c --- /dev/null +++ b/src/mlpack/tests/ann/layer/adaptive_mean_pooling.cpp @@ -0,0 +1,137 @@ +/** + * @file tests/ann/layer/adaptive_mean_pooling.cpp + * @author Marcus Edel + * @author Praveen Ch + * + * Tests the ann layer modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../test_catch_tools.hpp" +#include "../../catch.hpp" +#include "../../serialization.hpp" +#include "../ann_test_tools.hpp" + +using namespace mlpack; +using namespace mlpack::ann; + +/** + * Simple test for Adaptive pooling for Mean Pooling layer. + */ +TEST_CASE("AdaptiveMeanPoolingTestCase", "[ANNLayerTest]") +{ + // For rectangular input. + arma::mat input = arma::mat(12, 1); + arma::mat output, delta; + + input.zeros(); + input(0) = 1; + input(1) = 2; + input(2) = 3; + input(3) = input(8) = 7; + input(4) = 4; + input(5) = 5; + input(6) = input(7) = 6; + input(10) = 8; + input(11) = 9; + // Output-Size should be 2 x 2. + // Square output. + AdaptiveMeanPooling module1(2, 2); + output.set_size(4, 1); + module1.InputDimensions() = std::vector({ 4, 3 }); + module1.ComputeOutputDimensions(); + module1.Forward(input, output); + // Calculated using torch.nn.AdaptiveAvgPool2d(). + REQUIRE(arma::accu(output) == 19.75); + REQUIRE(output.n_elem == 4); + REQUIRE(output.n_cols == 1); + // Test the Backward Function. + delta.set_size(12, 1); + module1.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 19.75); + + // For Square input. + input = arma::mat(9, 1); + input.zeros(); + input(0) = 6; + input(1) = 3; + input(2) = 9; + input(3) = 3; + input(6) = 3; + // Output-Size should be 1 x 2. + // Rectangular output. + AdaptiveMeanPooling module2(1, 2); + output.set_size(2, 1); + module2.InputDimensions() = std::vector({ 3, 3 }); + module2.ComputeOutputDimensions(); + module2.Forward(input, output); + // Calculated using torch.nn.AdaptiveAvgPool2d(). + REQUIRE(arma::accu(output) == 4.5); + REQUIRE(output.n_elem == 2); + REQUIRE(output.n_cols == 1); + // Test the Backward Function. + delta.set_size(9, 1); + module2.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 4.50); + + // For Square input. + input = arma::mat(16, 1); + input.zeros(); + input(0) = 6; + input(1) = 3; + input(2) = 9; + input(4) = 3; + input(8) = 3; + // Output-Size should be 3 x 3. + // Square output. + AdaptiveMeanPooling module3(3, 3); + output.set_size(9, 1); + module3.InputDimensions() = std::vector({ 4, 4 }); + module3.ComputeOutputDimensions(); + module3.Forward(input, output); + // Calculated using torch.nn.AdaptiveAvgPool2d(). + REQUIRE(arma::accu(output) == 10.5); + REQUIRE(output.n_elem == 9); + REQUIRE(output.n_cols == 1); + // Test the Backward Function. + delta.set_size(16, 1); + module3.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 10.5); + + // For Rectangular input. + input = arma::mat(24, 1); + input.zeros(); + input(0) = 3; + input(1) = 3; + input(4) = 3; + // Output-Size should be 3 x 3. + // Square output. + AdaptiveMeanPooling module4(3, 3); + output.set_size(9, 1); + module4.InputDimensions() = std::vector({ 6, 4 }); + module4.ComputeOutputDimensions(); + module4.Forward(input, output); + // Calculated using torch.nn.AdaptiveAvgPool2d(). + REQUIRE(arma::accu(output) == 2.25); + REQUIRE(output.n_elem == 9); + REQUIRE(output.n_cols == 1); + // Test the Backward Function. + delta.set_size(24, 1); + module4.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 2.25); +} diff --git a/src/mlpack/tests/ann/layer/mean_pooling.cpp b/src/mlpack/tests/ann/layer/mean_pooling.cpp new file mode 100644 index 0000000000..f648f9ef13 --- /dev/null +++ b/src/mlpack/tests/ann/layer/mean_pooling.cpp @@ -0,0 +1,99 @@ +/** + * @file tests/ann/layer/mean_pooling.cpp + * @author Marcus Edel + * @author Praveen Ch + * + * Tests the ann layer modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../test_catch_tools.hpp" +#include "../../catch.hpp" +#include "../../serialization.hpp" +#include "../ann_test_tools.hpp" + +using namespace mlpack; +using namespace mlpack::ann; + +/** + * Simple test for Mean Pooling layer. + */ +TEST_CASE("MeanPoolingTestCase", "[ANNLayerTest]") +{ + // For rectangular input to pooling layers. + arma::mat input = arma::mat(28, 1); + input.zeros(); + input(0) = input(16) = 1; + input(1) = input(17) = 2; + input(2) = input(18) = 3; + input(3) = input(19) = 4; + input(4) = input(20) = 5; + input(5) = input(23) = 6; + input(6) = input(24) = 7; + input(14) = input(25) = 8; + input(15) = input(26) = 9; + + MeanPooling module1(2, 2, 2, 2, false); + MeanPooling module2(2, 2, 2, 2, true); + module1.InputDimensions() = std::vector({ 7, 4 }); + module1.ComputeOutputDimensions(); + module2.InputDimensions() = std::vector({ 7, 4 }); + module2.ComputeOutputDimensions(); + + // Calculated using torch.nn.MeanPool2d(). + arma::mat result1, result2; + result1 << 0.7500 << 4.2500 << arma::endr + << 1.7500 << 4.0000 << arma::endr + << 2.7500 << 6.0000 << arma::endr + << 3.5000 << 2.5000 << arma::endr; + + result2 << 0.7500 << 4.2500 << arma::endr + << 1.7500 << 4.0000 << arma::endr + << 2.7500 << 6.0000 << arma::endr; + + arma::mat output1, output2; + output1.set_size(8, 1); + output2.set_size(6, 1); + module1.Forward(input, output1); + REQUIRE(arma::accu(output1) == 25.5); + module2.Forward(input, output2); + REQUIRE(arma::accu(output2) == 19.5); + output1.reshape(4, 2); + output2.reshape(3, 2); + CheckMatrices(output1, result1, 1e-1); + CheckMatrices(output2, result2, 1e-1); + + arma::mat prevDelta1, prevDelta2; + prevDelta1 << 3.6000 << -0.9000 << arma::endr + << 3.6000 << -0.9000 << arma::endr + << 3.6000 << -0.9000 << arma::endr + << 3.6000 << -0.9000 << arma::endr; + + prevDelta2 << 3.6000 << -0.9000 << arma::endr + << 3.6000 << -0.9000 << arma::endr + << 3.6000 << -0.9000 << arma::endr; + arma::mat delta1, delta2; + delta1.set_size(28, 1); + delta2.set_size(28, 1); + prevDelta1.reshape(8, 1); + prevDelta2.reshape(6, 1); + module1.Backward(input, prevDelta1, delta1); + REQUIRE(arma::accu(delta1) == Approx(10.8).epsilon(1e-3)); + module2.Backward(input, prevDelta2, delta2); + REQUIRE(arma::accu(delta2) == Approx(8.1).epsilon(1e-3)); +} diff --git a/src/mlpack/tests/ann/layer/softmax.cpp b/src/mlpack/tests/ann/layer/softmax.cpp new file mode 100644 index 0000000000..afbf517112 --- /dev/null +++ b/src/mlpack/tests/ann/layer/softmax.cpp @@ -0,0 +1,94 @@ +/** + * @file tests/ann/layer/softmax.cpp + * @author Marcus Edel + * @author Praveen Ch + * + * Tests the ann layer modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../test_catch_tools.hpp" +#include "../../catch.hpp" +#include "../../serialization.hpp" +#include "../ann_test_tools.hpp" + +using namespace mlpack; +using namespace mlpack::ann; + +/** + * Simple Softmax module test. + */ +TEST_CASE("SimpleSoftmaxLayerTest", "[ANNLayerTest]") +{ + arma::mat input, output, gy, g; + Softmax module; + + // Test the forward function. + input = arma::mat("1.7; 3.6"); + module.Forward(input, output); + REQUIRE(arma::accu(arma::abs(arma::mat("0.130108; 0.869892") - output)) == + Approx(0.0).margin(1e-4)); + + // Test the backward function. + gy = arma::zeros(input.n_rows, input.n_cols); + gy(0) = 1; + module.Backward(output, gy, g); + REQUIRE(arma::accu(arma::abs(arma::mat("0.11318; -0.11318") - g)) == + Approx(0.0).margin(1e-04)); +} + +/** + * Softmax layer numerical gradient test. + */ +TEST_CASE("GradientSoftmaxTest", "[ANNLayerTest]") +{ + // Softmax function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(10, 1)), + target(arma::mat("1; 0")) + { + model = new FFN; + model->ResetData(input, target); + model->Add(10); + model->Add(); + model->Add(2); + model->Add(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + REQUIRE(CheckGradient(function) <= 1e-4); +} From 0d93611ee814813dbc17c005c26b5b867507067e Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sat, 11 Jun 2022 12:23:23 -0400 Subject: [PATCH 08/39] Adjust tolerances on SimpleAlphaDropoutLayerTest. --- src/mlpack/tests/ann/layer/alpha_dropout.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/tests/ann/layer/alpha_dropout.cpp b/src/mlpack/tests/ann/layer/alpha_dropout.cpp index 066b940e33..ffe28d1ec9 100644 --- a/src/mlpack/tests/ann/layer/alpha_dropout.cpp +++ b/src/mlpack/tests/ann/layer/alpha_dropout.cpp @@ -44,16 +44,16 @@ TEST_CASE("SimpleAlphaDropoutLayerTest", "[ANNLayerTest]") module.Forward(input, output); // Check whether mean remains nearly same. REQUIRE(arma::as_scalar(arma::abs(arma::mean(input) - arma::mean(output))) <= - 0.1); + 0.15); // Check whether variance remains nearly same. REQUIRE(arma::as_scalar(arma::abs(arma::var(input) - arma::var(output))) <= - 0.1); + 0.15); // Test the Backward function when training phase. arma::mat delta; module.Backward(input, input, delta); - REQUIRE(arma::as_scalar(arma::abs(arma::mean(delta) - 0)) <= 0.05); + REQUIRE(arma::as_scalar(arma::abs(arma::mean(delta) - 0)) <= 0.1); // Test the Forward function when testing phase. module.Training() = false; From b892828be39e76e3c17df06bf89cb7ece322b0d6 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 20 Jun 2022 14:33:24 -0400 Subject: [PATCH 09/39] Adapt several files to header-only. --- src/mlpack/core/data/CMakeLists.txt | 1 + .../core/data/check_categorical_param.hpp | 34 +++++ src/mlpack/core/data/dataset_mapper.hpp | 3 + src/mlpack/core/data/dataset_mapper_impl.hpp | 6 + src/mlpack/core/util/CMakeLists.txt | 8 +- src/mlpack/core/util/backtrace.hpp | 53 ++++++- .../{backtrace.cpp => backtrace_impl.hpp} | 72 +++------ src/mlpack/core/util/forward.hpp | 37 +++++ src/mlpack/core/util/param.hpp | 14 +- src/mlpack/core/util/params.cpp | 144 ------------------ src/mlpack/core/util/params.hpp | 1 + src/mlpack/core/util/params_impl.hpp | 131 ++++++++++++++++ src/mlpack/core/util/timers.hpp | 3 + .../core/util/{timers.cpp => timers_impl.hpp} | 122 ++++++++------- src/mlpack/core/util/version.hpp | 5 +- .../util/{version.cpp => version_impl.hpp} | 10 +- 16 files changed, 367 insertions(+), 277 deletions(-) create mode 100644 src/mlpack/core/data/check_categorical_param.hpp rename src/mlpack/core/util/{backtrace.cpp => backtrace_impl.hpp} (66%) create mode 100644 src/mlpack/core/util/forward.hpp delete mode 100644 src/mlpack/core/util/params.cpp rename src/mlpack/core/util/{timers.cpp => timers_impl.hpp} (50%) rename src/mlpack/core/util/{version.cpp => version_impl.hpp} (84%) diff --git a/src/mlpack/core/data/CMakeLists.txt b/src/mlpack/core/data/CMakeLists.txt index 0b83444c00..38a9b6ec61 100644 --- a/src/mlpack/core/data/CMakeLists.txt +++ b/src/mlpack/core/data/CMakeLists.txt @@ -1,6 +1,7 @@ # Define the files that we need to compile. # Anything not in this list will not be compiled into mlpack. set(SOURCES + check_categorical_param.hpp dataset_mapper.hpp dataset_mapper_impl.hpp detect_file_type.hpp diff --git a/src/mlpack/core/data/check_categorical_param.hpp b/src/mlpack/core/data/check_categorical_param.hpp new file mode 100644 index 0000000000..06085e7261 --- /dev/null +++ b/src/mlpack/core/data/check_categorical_param.hpp @@ -0,0 +1,34 @@ +/** + * @file check_categorical_param.hpp + * @author Ryan Curtin + * + * This file provides an implementation of a simple function to check the values + * of a categorical parameter. It cannot be defined in util/, since the + * DatasetMapper class is not fully defined when that is included.. + */ +#ifndef MLPACK_CORE_DATA_CHECK_CATEGORICAL_PARAM_HPP +#define MLPACK_CORE_DATA_CHECK_CATEGORICAL_PARAM_HPP + +namespace mlpack { +namespace data { + +inline void CheckCategoricalParam(util::Params& params, + const std::string& paramName) +{ + typedef typename std::tuple TupleType; + arma::mat& matrix = std::get<1>(params.Get(paramName)); + + // This comes from Params::CheckInputMatrix(). + const std::string errMsg1 = "The input '" + paramName + "' has NaN values."; + const std::string errMsg2 = "The input '" + paramName + "' has Inf values."; + + if (matrix.has_nan()) + Log::Fatal << errMsg1 << std::endl; + if (matrix.has_inf()) + Log::Fatal << errMsg2 << std::endl; +} + +} // namespace data +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/data/dataset_mapper.hpp b/src/mlpack/core/data/dataset_mapper.hpp index ec959e70b5..893385e0cf 100644 --- a/src/mlpack/core/data/dataset_mapper.hpp +++ b/src/mlpack/core/data/dataset_mapper.hpp @@ -200,4 +200,7 @@ using DatasetInfo = DatasetMapper; #include "dataset_mapper_impl.hpp" +// Also include utility function. +#include "check_categorical_param.hpp" + #endif diff --git a/src/mlpack/core/data/dataset_mapper_impl.hpp b/src/mlpack/core/data/dataset_mapper_impl.hpp index 7422adc673..e5574b58e2 100644 --- a/src/mlpack/core/data/dataset_mapper_impl.hpp +++ b/src/mlpack/core/data/dataset_mapper_impl.hpp @@ -248,6 +248,12 @@ inline void DatasetMapper::Policy(PolicyType&& policy) this->policy = std::forward(policy); } +// Implementation of utility function needed by Params class. +inline arma::mat& GetMatrix(std::tuple& t) +{ + return std::get<1>(t); +} + } // namespace data } // namespace mlpack diff --git a/src/mlpack/core/util/CMakeLists.txt b/src/mlpack/core/util/CMakeLists.txt index 4ffe767d47..f18c4d790b 100644 --- a/src/mlpack/core/util/CMakeLists.txt +++ b/src/mlpack/core/util/CMakeLists.txt @@ -5,8 +5,9 @@ set(SOURCES arma_config.hpp arma_config_check.hpp backtrace.hpp - backtrace.cpp + backtrace_impl.hpp binding_details.hpp + forward.hpp io.hpp io.cpp deprecated.hpp @@ -22,7 +23,6 @@ set(SOURCES param_data.hpp params.hpp params_impl.hpp - params.cpp prefixedoutstream.hpp prefixedoutstream.cpp prefixedoutstream_impl.hpp @@ -32,10 +32,10 @@ set(SOURCES sfinae_utility.hpp singletons.cpp timers.hpp - timers.cpp + timers_impl.hpp to_lower.hpp version.hpp - version.cpp + version_impl.hpp ) # add directory name to sources diff --git a/src/mlpack/core/util/backtrace.hpp b/src/mlpack/core/util/backtrace.hpp index 181240f8f2..ed58a839b7 100644 --- a/src/mlpack/core/util/backtrace.hpp +++ b/src/mlpack/core/util/backtrace.hpp @@ -15,6 +15,37 @@ #include #include +#ifdef HAS_BFD_DL + #include + #include + #include + #include + + // Some versions of libbfd require PACKAGE and PACKAGE_VERSION to be set in + // order for the include to not fail. For more information: + // https://github.com/mlpack/mlpack/issues/574 + #ifndef PACKAGE + #define PACKAGE + #ifndef PACKAGE_VERSION + #define PACKAGE_VERSION + #include + #undef PACKAGE_VERSION + #else + #include + #endif + #undef PACKAGE + #else + #ifndef PACKAGE_VERSION + #define PACKAGE_VERSION + #include + #undef PACKAGE_VERSION + #else + #include + #endif + #endif + #include +#endif + namespace mlpack { /** @@ -71,17 +102,17 @@ class Backtrace * * @param maxDepth Maximum depth of backtrace. Default 32 steps. */ - static void GetAddress(int maxDepth); + void GetAddress(int maxDepth); /** * Decodes file name, function & line number. * * @param address Address of traced frame. */ - static void DecodeAddress(long address); + void DecodeAddress(long address); //! Demangles function name. - static void DemangleFunction(); + void DemangleFunction(); //! Backtrace datastructure. struct Frames @@ -90,12 +121,22 @@ class Backtrace const char* function; const char* file; unsigned line; - } static frame; + }; - //! A vector for all the backtrace information. - static std::vector stack; + Frames frame; + std::vector stack; + +#ifdef HAS_BFD_DL + // Binary File Descriptor objects. + bfd* abfd; // Descriptor datastructure. + asymbol **syms; // Symbols datastructure. + asection *text; // Strings datastructure. +#endif }; }; // namespace mlpack +// Include implementation. +#include "backtrace_impl.hpp" + #endif diff --git a/src/mlpack/core/util/backtrace.cpp b/src/mlpack/core/util/backtrace_impl.hpp similarity index 66% rename from src/mlpack/core/util/backtrace.cpp rename to src/mlpack/core/util/backtrace_impl.hpp index 3bdf7c4f29..a3eea2bf7f 100644 --- a/src/mlpack/core/util/backtrace.cpp +++ b/src/mlpack/core/util/backtrace_impl.hpp @@ -1,5 +1,5 @@ /** - * @file core/util/backtrace.cpp + * @file core/util/backtrace_impl.hpp * @author Grzegorz Krajewski * * Implementation of the Backtrace class. @@ -11,74 +11,36 @@ */ #include -#ifdef HAS_BFD_DL - #include - #include - #include - #include - - // Some versions of libbfd require PACKAGE and PACKAGE_VERSION to be set in - // order for the include to not fail. For more information: - // https://github.com/mlpack/mlpack/issues/574 - #ifndef PACKAGE - #define PACKAGE - #ifndef PACKAGE_VERSION - #define PACKAGE_VERSION - #include - #undef PACKAGE_VERSION - #else - #include - #endif - #undef PACKAGE - #else - #ifndef PACKAGE_VERSION - #define PACKAGE_VERSION - #include - #undef PACKAGE_VERSION - #else - #include - #endif - #endif - #include -#endif - #include "backtrace.hpp" #include "log.hpp" -using namespace mlpack; - -// Initialize Backtrace static inctances. -Backtrace::Frames Backtrace::frame; -std::vector Backtrace::stack; +namespace mlpack { #ifdef HAS_BFD_DL -// Binary File Descriptor objects. -bfd* abfd = 0; // Descriptor datastructure. -asymbol **syms = 0; // Symbols datastructure. -asection *text = 0; // Strings datastructure. -#endif - -#ifdef HAS_BFD_DL -Backtrace::Backtrace(int maxDepth) +inline Backtrace::Backtrace(int maxDepth) { frame.address = NULL; frame.function = "0"; frame.file = "0"; frame.line = 0; + abfd = 0; + syms = 0; + text = 0; + stack.clear(); GetAddress(maxDepth); } #else -Backtrace::Backtrace() +inline Backtrace::Backtrace() { // Dummy constructor } #endif #ifdef HAS_BFD_DL -void Backtrace::GetAddress(int maxDepth) +inline void Backtrace::GetAddress(int maxDepth) { void* trace[maxDepth]; int stackDepth = backtrace(trace, maxDepth); @@ -100,7 +62,7 @@ void Backtrace::GetAddress(int maxDepth) } } -void Backtrace::DecodeAddress(long addr) +inline void Backtrace::DecodeAddress(long addr) { // Check to see if there is anything to descript. If it doesn't, we'll // dump running program. @@ -147,7 +109,7 @@ void Backtrace::DecodeAddress(long addr) } } -void Backtrace::DemangleFunction() +inline void Backtrace::DemangleFunction() { int status; char* tmp = abi::__cxa_demangle(frame.function, 0, 0, &status); @@ -160,12 +122,12 @@ void Backtrace::DemangleFunction() } } #else -void Backtrace::GetAddress(int /* maxDepth */) { } -void Backtrace::DecodeAddress(long /* address */) { } -void Backtrace::DemangleFunction() { } +inline void Backtrace::GetAddress(int /* maxDepth */) { } +inline void Backtrace::DecodeAddress(long /* address */) { } +inline void Backtrace::DemangleFunction() { } #endif -std::string Backtrace::ToString() +inline std::string Backtrace::ToString() { std::string stackStr; @@ -198,8 +160,10 @@ std::string Backtrace::ToString() it.str(""); } #else - stackStr = "[bt]: No backtrace for this OS. Work in progress."; + stackStr = "[bt]: No backtrace for this OS."; #endif return stackStr; } + +} // namespace mlpack diff --git a/src/mlpack/core/util/forward.hpp b/src/mlpack/core/util/forward.hpp new file mode 100644 index 0000000000..46200721bb --- /dev/null +++ b/src/mlpack/core/util/forward.hpp @@ -0,0 +1,37 @@ +/** + * @file forward.hpp + * @author Ryan Curtin + * + * Forward declaration of components from other subdirectories necessary for + * various util implementations. + */ +#ifndef MLPACK_CORE_UTIL_FORWARD_HPP +#define MLPACK_CORE_UTIL_FORWARD_HPP + +// Required forward declarations. +namespace mlpack { + +namespace util { + +class Params; + +} // namespace util + +namespace data { + +class IncrementPolicy; + +template +class DatasetMapper; + +using DatasetInfo = DatasetMapper; + +// This is a forward declaration of a function that just calls std::get(); but, +// we cannot use std::get directly because we have only forward-declared +// DatasetInfo. +void CheckCategoricalParam(util::Params& p, const std::string& paramName); + +} // namespace data +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/util/param.hpp b/src/mlpack/core/util/param.hpp index c2dbe7e2aa..6ca2a8ed6c 100644 --- a/src/mlpack/core/util/param.hpp +++ b/src/mlpack/core/util/param.hpp @@ -15,19 +15,7 @@ #ifndef MLPACK_CORE_UTIL_PARAM_HPP #define MLPACK_CORE_UTIL_PARAM_HPP -// Required forward declarations. -namespace mlpack { -namespace data { - -class IncrementPolicy; - -template -class DatasetMapper; - -using DatasetInfo = DatasetMapper; - -} // namespace data -} // namespace mlpack +#include "forward.hpp" /** * @cond diff --git a/src/mlpack/core/util/params.cpp b/src/mlpack/core/util/params.cpp deleted file mode 100644 index cfd9b0fd11..0000000000 --- a/src/mlpack/core/util/params.cpp +++ /dev/null @@ -1,144 +0,0 @@ -/** - * @file params.cpp - * @author Ryan Curtin - * - * Implementation of functions in the Param class. - */ -#include "params.hpp" -#include - -namespace mlpack { -namespace util { - -Params::Params(const std::map& aliases, - const std::map& parameters, - Params::FunctionMapType& functionMap, - const std::string& bindingName, - const BindingDetails& doc) : - // Copy all the given inputs. - aliases(aliases), - parameters(parameters), - functionMap(functionMap), - bindingName(bindingName), - doc(doc) -{ - // Nothing to do. -} - -Params::Params() -{ - // Nothing to do. -} - -/** - * Return `true` if the specified parameter was given. - * - * @param identifier The name of the parameter in question. - */ -bool Params::Has(const std::string& key) const -{ - std::string usedKey = key; - - if (!parameters.count(key)) - { - // Check any aliases, but only after we are sure the actual option as given - // does not exist. - // TODO: can we isolate alias support inside of the CLI binding code? - if (key.length() == 1 && aliases.count(key[0])) - usedKey = aliases.at(key[0]); - - if (!parameters.count(usedKey)) - { - Log::Fatal << "Parameter '" << key << "' does not exist in this " - << "program." << std::endl; - } - } - const std::string& checkKey = usedKey; - - return (parameters.at(checkKey).wasPassed > 0); -} - -/** - * Given two (matrix) parameters, ensure that the first is an in-place copy of - * the second. This will generally do nothing (as the bindings already do - * this automatically), except for command-line bindings, where we need to - * ensure that the output filename is the same as the input filename. - * - * @param outputParamName Name of output (matrix) parameter. - * @param inputParamName Name of input (matrix) parameter. - */ -void Params::MakeInPlaceCopy(const std::string& outputParamName, - const std::string& inputParamName) -{ - if (!parameters.count(outputParamName)) - Log::Fatal << "Unknown parameter '" << outputParamName << "'!" << std::endl; - if (!parameters.count(inputParamName)) - Log::Fatal << "Unknown parameter '" << inputParamName << "'!" << std::endl; - - ParamData& output = parameters[outputParamName]; - ParamData& input = parameters[inputParamName]; - - if (output.cppType != input.cppType) - { - Log::Fatal << "Cannot call MakeInPlaceCopy() with different types (" - << output.cppType << " and " << input.cppType << ")!" << std::endl; - } - - // Is there a function to do this? - if (functionMap[output.tname].count("InPlaceCopy") != 0) - { - functionMap[output.tname]["InPlaceCopy"](output, (void*) &input, NULL); - } -} - -/** - * Set the particular parameter as passed. - * - * @param identifier The name of the parameter to set as passed. - */ -void Params::SetPassed(const std::string& name) -{ - if (parameters.count(name) == 0) - { - throw std::invalid_argument("Params::SetPassed(): parameter " + name + - " not known for binding " + bindingName + "!"); - } - - // Set passed to true. - parameters[name].wasPassed = true; -} - -/** - * Check all input matrices for NaN and inf values, and throw an exception if - * any are found. - */ -void Params::CheckInputMatrices() -{ - typedef typename std::tuple TupleType; - std::map::iterator itr; - - for (itr = parameters.begin(); itr != parameters.end(); ++itr) - { - std::string paramName = itr->first; - std::string paramType = itr->second.cppType; - if (paramType == "arma::mat") - { - CheckInputMatrix(Get(paramName), paramName); - } - else if (paramType == "arma::vec") - { - CheckInputMatrix(Get(paramName), paramName); - } - else if (paramType == "arma::rowvec") - { - CheckInputMatrix(Get(paramName), paramName); - } - else if (paramType == "std::tuple") - { - CheckInputMatrix(std::get<1>(Get(paramName)), paramName); - } - } -} - -} // namespace util -} // namespace mlpack diff --git a/src/mlpack/core/util/params.hpp b/src/mlpack/core/util/params.hpp index 00cd31a908..c6bb1596a7 100644 --- a/src/mlpack/core/util/params.hpp +++ b/src/mlpack/core/util/params.hpp @@ -7,6 +7,7 @@ #ifndef MLPACK_CORE_UTIL_PARAMS_HPP #define MLPACK_CORE_UTIL_PARAMS_HPP +#include "forward.hpp" #include "param_data.hpp" #include "binding_details.hpp" diff --git a/src/mlpack/core/util/params_impl.hpp b/src/mlpack/core/util/params_impl.hpp index 79d29de1c7..1c53205b94 100644 --- a/src/mlpack/core/util/params_impl.hpp +++ b/src/mlpack/core/util/params_impl.hpp @@ -14,6 +14,54 @@ namespace mlpack { namespace util { +inline Params::Params(const std::map& aliases, + const std::map& parameters, + Params::FunctionMapType& functionMap, + const std::string& bindingName, + const BindingDetails& doc) : + // Copy all the given inputs. + aliases(aliases), + parameters(parameters), + functionMap(functionMap), + bindingName(bindingName), + doc(doc) +{ + // Nothing to do. +} + +inline Params::Params() +{ + // Nothing to do. +} + +/** + * Return `true` if the specified parameter was given. + * + * @param identifier The name of the parameter in question. + */ +inline bool Params::Has(const std::string& key) const +{ + std::string usedKey = key; + + if (!parameters.count(key)) + { + // Check any aliases, but only after we are sure the actual option as given + // does not exist. + // TODO: can we isolate alias support inside of the CLI binding code? + if (key.length() == 1 && aliases.count(key[0])) + usedKey = aliases.at(key[0]); + + if (!parameters.count(usedKey)) + { + Log::Fatal << "Parameter '" << key << "' does not exist in this " + << "program." << std::endl; + } + } + const std::string& checkKey = usedKey; + + return (parameters.at(checkKey).wasPassed > 0); +} + /** * Get the value of type T found for the parameter specified by `identifier`. * You can set the value using this reference safely. @@ -156,6 +204,89 @@ void Params::CheckInputMatrix(const T& matrix, const std::string& identifier) Log::Fatal << errMsg2 << std::endl; } +/** + * Given two (matrix) parameters, ensure that the first is an in-place copy of + * the second. This will generally do nothing (as the bindings already do + * this automatically), except for command-line bindings, where we need to + * ensure that the output filename is the same as the input filename. + * + * @param outputParamName Name of output (matrix) parameter. + * @param inputParamName Name of input (matrix) parameter. + */ +inline void Params::MakeInPlaceCopy(const std::string& outputParamName, + const std::string& inputParamName) +{ + if (!parameters.count(outputParamName)) + Log::Fatal << "Unknown parameter '" << outputParamName << "'!" << std::endl; + if (!parameters.count(inputParamName)) + Log::Fatal << "Unknown parameter '" << inputParamName << "'!" << std::endl; + + ParamData& output = parameters[outputParamName]; + ParamData& input = parameters[inputParamName]; + + if (output.cppType != input.cppType) + { + Log::Fatal << "Cannot call MakeInPlaceCopy() with different types (" + << output.cppType << " and " << input.cppType << ")!" << std::endl; + } + + // Is there a function to do this? + if (functionMap[output.tname].count("InPlaceCopy") != 0) + { + functionMap[output.tname]["InPlaceCopy"](output, (void*) &input, NULL); + } +} + +/** + * Set the particular parameter as passed. + * + * @param identifier The name of the parameter to set as passed. + */ +inline void Params::SetPassed(const std::string& name) +{ + if (parameters.count(name) == 0) + { + throw std::invalid_argument("Params::SetPassed(): parameter " + name + + " not known for binding " + bindingName + "!"); + } + + // Set passed to true. + parameters[name].wasPassed = true; +} + +/** + * Check all input matrices for NaN and inf values, and throw an exception if + * any are found. + */ +inline void Params::CheckInputMatrices() +{ + std::map::iterator itr; + + for (itr = parameters.begin(); itr != parameters.end(); ++itr) + { + std::string paramName = itr->first; + std::string paramType = itr->second.cppType; + if (paramType == "arma::mat") + { + CheckInputMatrix(Get(paramName), paramName); + } + else if (paramType == "arma::vec") + { + CheckInputMatrix(Get(paramName), paramName); + } + else if (paramType == "arma::rowvec") + { + CheckInputMatrix(Get(paramName), paramName); + } + else if (paramType == "std::tuple") + { + // Note that CheckCategoricalParam() is a utility function that must be + // defined after DatasetInfo is fully defined. + data::CheckCategoricalParam(*this, paramName); + } + } +} + } // namespace util } // namespace mlpack diff --git a/src/mlpack/core/util/timers.hpp b/src/mlpack/core/util/timers.hpp index 177850de52..ee7279c004 100644 --- a/src/mlpack/core/util/timers.hpp +++ b/src/mlpack/core/util/timers.hpp @@ -182,4 +182,7 @@ class Timers } // namespace util } // namespace mlpack +// Include implementation. +#include "timers_impl.hpp" + #endif // MLPACK_CORE_UTILITIES_TIMERS_HPP diff --git a/src/mlpack/core/util/timers.cpp b/src/mlpack/core/util/timers_impl.hpp similarity index 50% rename from src/mlpack/core/util/timers.cpp rename to src/mlpack/core/util/timers_impl.hpp index b93f5731fe..2ca43f80ca 100644 --- a/src/mlpack/core/util/timers.cpp +++ b/src/mlpack/core/util/timers_impl.hpp @@ -1,5 +1,5 @@ /** - * @file core/util/timers.cpp + * @file core/util/timers_impl.hpp * @author Matthew Amidon * @author Marcus Edel * @author Ryan Curtin @@ -18,100 +18,104 @@ #include #include -using namespace mlpack; -using namespace mlpack::util; -using namespace std; -using namespace chrono; +namespace mlpack { /** * Start the given timer. */ -void Timer::Start(const string& name) +inline void Timer::Start(const std::string& name) { - IO::GetSingleton().timer.Start(name, this_thread::get_id()); + IO::GetSingleton().timer.Start(name, std::this_thread::get_id()); } /** * Stop the given timer. */ -void Timer::Stop(const string& name) +inline void Timer::Stop(const std::string& name) { - IO::GetSingleton().timer.Stop(name, this_thread::get_id()); + IO::GetSingleton().timer.Stop(name, std::this_thread::get_id()); } /** * Get the given timer, summing over all threads. */ -microseconds Timer::Get(const string& name) +inline std::chrono::microseconds Timer::Get(const std::string& name) { return IO::GetSingleton().timer.Get(name); } // Enable timing. -void Timer::EnableTiming() +inline void Timer::EnableTiming() { IO::GetSingleton().timer.Enabled() = true; } // Disable timing. -void Timer::DisableTiming() +inline void Timer::DisableTiming() { IO::GetSingleton().timer.Enabled() = false; } // Reset all timers. Save state of enabled. -void Timer::ResetAll() +inline void Timer::ResetAll() { IO::GetSingleton().timer.Reset(); } -std::map Timer::GetAllTimers() +inline std::map Timer::GetAllTimers() { return IO::GetSingleton().timer.GetAllTimers(); } +namespace util { + // Reset a Timers object. -void Timers::Reset() +inline void Timers::Reset() { - lock_guard lock(timersMutex); + std::lock_guard lock(timersMutex); timers.clear(); timerStartTime.clear(); } -map Timers::GetAllTimers() +inline std::map Timers::GetAllTimers() { // Make a copy of the timer. - lock_guard lock(timersMutex); + std::lock_guard lock(timersMutex); return timers; } -microseconds Timers::Get(const string& timerName) +inline std::chrono::microseconds Timers::Get(const std::string& timerName) { if (!enabled) - return microseconds(0); + return std::chrono::microseconds(0); - lock_guard lock(timersMutex); + std::lock_guard lock(timersMutex); return timers[timerName]; } -std::string Timers::Print(const microseconds& totalDuration) +inline std::string Timers::Print(const std::chrono::microseconds& totalDuration) { // Convert microseconds to seconds. - seconds totalDurationSec = duration_cast(totalDuration); - microseconds totalDurationMicroSec = - duration_cast(totalDuration % seconds(1)); + std::chrono::seconds totalDurationSec = + std::chrono::duration_cast(totalDuration); + std::chrono::microseconds totalDurationMicroSec = + std::chrono::duration_cast( + totalDuration % std::chrono::seconds(1)); std::ostringstream oss; - oss << totalDurationSec.count() << "." << setw(6) - << setfill('0') << totalDurationMicroSec.count() << "s"; + oss << totalDurationSec.count() << "." << std::setw(6) + << std::setfill('0') << totalDurationMicroSec.count() << "s"; // Also output convenient day/hr/min/sec. // The following line is a custom duration for a day. - typedef duration> days; - days d = duration_cast(totalDuration); - hours h = duration_cast(totalDuration % days(1)); - minutes m = duration_cast(totalDuration % hours(1)); - seconds s = duration_cast(totalDuration % minutes(1)); + typedef std::chrono::duration> days; + days d = std::chrono::duration_cast(totalDuration); + std::chrono::hours h = std::chrono::duration_cast( + totalDuration % days(1)); + std::chrono::minutes m = std::chrono::duration_cast( + totalDuration % std::chrono::hours(1)); + std::chrono::seconds s = std::chrono::duration_cast( + totalDuration % std::chrono::minutes(1)); // No output if it didn't even take a minute. if (!(d.count() == 0 && h.count() == 0 && m.count() == 0)) { @@ -145,87 +149,99 @@ std::string Timers::Print(const microseconds& totalDuration) { if (output) oss << ", "; - oss << s.count() << "." << setw(1) + oss << s.count() << "." << std::setw(1) << (totalDurationMicroSec.count() / 100000) << " secs"; } oss << ")"; } - oss << endl; + oss << std::endl; return oss.str(); } -void Timers::StopAllTimers() +inline void Timers::StopAllTimers() { // Terminate the program timers. Don't use StopTimer() since that modifies // the map and would invalidate our iterators. - lock_guard lock(timersMutex); + std::lock_guard lock(timersMutex); - high_resolution_clock::time_point currTime = high_resolution_clock::now(); + std::chrono::high_resolution_clock::time_point currTime = + std::chrono::high_resolution_clock::now(); for (auto it : timerStartTime) + { for (auto it2 : it.second) - timers[it2.first] += duration_cast(currTime - it2.second); + { + timers[it2.first] += + std::chrono::duration_cast( + currTime - it2.second); + } + } // If all timers are stopped, we can clear the maps. timerStartTime.clear(); } -void Timers::Start(const string& timerName, - const thread::id& threadId) +inline void Timers::Start(const std::string& timerName, + const std::thread::id& threadId) { // Don't do anything if we aren't timing. if (!enabled) return; - lock_guard lock(timersMutex); + std::lock_guard lock(timersMutex); if ((timerStartTime.count(threadId) > 0) && (timerStartTime[threadId].count(timerName))) { - ostringstream error; + std::ostringstream error; error << "Timer::Start(): timer '" << timerName << "' has already been started"; - throw runtime_error(error.str()); + throw std::runtime_error(error.str()); } - high_resolution_clock::time_point currTime = high_resolution_clock::now(); + std::chrono::high_resolution_clock::time_point currTime = + std::chrono::high_resolution_clock::now(); // If the timer is added for the first time. if (timers.count(timerName) == 0) { - timers[timerName] = (microseconds) 0; + timers[timerName] = (std::chrono::microseconds) 0; } timerStartTime[threadId][timerName] = currTime; } -void Timers::Stop(const string& timerName, - const thread::id& threadId) +inline void Timers::Stop(const std::string& timerName, + const std::thread::id& threadId) { // Don't do anything if we aren't timing. if (!enabled) return; - lock_guard lock(timersMutex); + std::lock_guard lock(timersMutex); if ((timerStartTime.count(threadId) == 0) || (timerStartTime[threadId].count(timerName) == 0)) { - ostringstream error; + std::ostringstream error; error << "Timer::Stop(): no timer with name '" << timerName << "' currently running"; - throw runtime_error(error.str()); + throw std::runtime_error(error.str()); } - high_resolution_clock::time_point currTime = high_resolution_clock::now(); + std::chrono::high_resolution_clock::time_point currTime = + std::chrono::high_resolution_clock::now(); // Calculate the delta time. - timers[timerName] += duration_cast(currTime - - timerStartTime[threadId][timerName]); + timers[timerName] += std::chrono::duration_cast( + currTime - timerStartTime[threadId][timerName]); // Remove the entries. timerStartTime[threadId].erase(timerName); if (timerStartTime[threadId].empty()) timerStartTime.erase(threadId); } + +} // namespace util +} // namespace mlpack diff --git a/src/mlpack/core/util/version.hpp b/src/mlpack/core/util/version.hpp index 5292b64999..474c04225a 100644 --- a/src/mlpack/core/util/version.hpp +++ b/src/mlpack/core/util/version.hpp @@ -28,9 +28,12 @@ namespace util { * This will return either "mlpack x.y.z" or "mlpack master-XXXXXXX" depending on * whether or not this is a stable version of mlpack or a git repository. */ -std::string GetVersion(); +inline std::string GetVersion(); } // namespace util } // namespace mlpack +// Include implementation. +#include "version_impl.hpp" + #endif diff --git a/src/mlpack/core/util/version.cpp b/src/mlpack/core/util/version_impl.hpp similarity index 84% rename from src/mlpack/core/util/version.cpp rename to src/mlpack/core/util/version_impl.hpp index 2e0dfd1cf2..23f29554f3 100644 --- a/src/mlpack/core/util/version.cpp +++ b/src/mlpack/core/util/version_impl.hpp @@ -1,5 +1,5 @@ /** - * @file core/util/version.cpp + * @file core/util/version_impl.hpp * @author Ryan Curtin * * The implementation of GetVersion(). @@ -13,9 +13,12 @@ #include +namespace mlpack { +namespace util { + // If we are not a git revision, just use the macros to assemble the version // name. -std::string mlpack::util::GetVersion() +inline std::string GetVersion() { #ifndef MLPACK_GIT_VERSION std::stringstream o; @@ -28,3 +31,6 @@ std::string mlpack::util::GetVersion() #include "gitversion.hpp" #endif } + +} // namespace util +} // namespace mlpack From 19f4b2284040a8d90d2756e0890035f57c2fdb5e Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 20 Jun 2022 15:00:36 -0400 Subject: [PATCH 10/39] Move PrefixedOutStream implementations to be inline. --- src/mlpack/core/util/CMakeLists.txt | 1 - src/mlpack/core/util/prefixedoutstream.cpp | 127 ------------------ .../core/util/prefixedoutstream_impl.hpp | 110 +++++++++++++++ 3 files changed, 110 insertions(+), 128 deletions(-) delete mode 100644 src/mlpack/core/util/prefixedoutstream.cpp diff --git a/src/mlpack/core/util/CMakeLists.txt b/src/mlpack/core/util/CMakeLists.txt index f18c4d790b..87a2786af1 100644 --- a/src/mlpack/core/util/CMakeLists.txt +++ b/src/mlpack/core/util/CMakeLists.txt @@ -24,7 +24,6 @@ set(SOURCES params.hpp params_impl.hpp prefixedoutstream.hpp - prefixedoutstream.cpp prefixedoutstream_impl.hpp program_doc.hpp program_doc.cpp diff --git a/src/mlpack/core/util/prefixedoutstream.cpp b/src/mlpack/core/util/prefixedoutstream.cpp deleted file mode 100644 index 283cdcd10f..0000000000 --- a/src/mlpack/core/util/prefixedoutstream.cpp +++ /dev/null @@ -1,127 +0,0 @@ -/** - * @file core/util/prefixedoutstream.cpp - * @author Ryan Curtin - * @author Matthew Amidon - * - * Implementation of PrefixedOutStream methods. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include - -#include "prefixedoutstream.hpp" - -using namespace mlpack::util; - -/** - * These are all necessary because gcc's template mechanism does not seem smart - * enough to figure out what I want to pass into operator<< without these. That - * may not be the actual case, but it works when these is here. - */ - -PrefixedOutStream& PrefixedOutStream::operator<<(bool val) -{ - BaseLogic(val); - return *this; -} - -PrefixedOutStream& PrefixedOutStream::operator<<(short val) -{ - BaseLogic(val); - return *this; -} - -PrefixedOutStream& PrefixedOutStream::operator<<(unsigned short val) -{ - BaseLogic(val); - return *this; -} - -PrefixedOutStream& PrefixedOutStream::operator<<(int val) -{ - BaseLogic(val); - return *this; -} - -PrefixedOutStream& PrefixedOutStream::operator<<(unsigned int val) -{ - BaseLogic(val); - return *this; -} - -PrefixedOutStream& PrefixedOutStream::operator<<(long val) -{ - BaseLogic(val); - return *this; -} - -PrefixedOutStream& PrefixedOutStream::operator<<(unsigned long val) -{ - BaseLogic(val); - return *this; -} - -PrefixedOutStream& PrefixedOutStream::operator<<(float val) -{ - BaseLogic(val); - return *this; -} - -PrefixedOutStream& PrefixedOutStream::operator<<(double val) -{ - BaseLogic(val); - return *this; -} - -PrefixedOutStream& PrefixedOutStream::operator<<(long double val) -{ - BaseLogic(val); - return *this; -} - -PrefixedOutStream& PrefixedOutStream::operator<<(void* val) -{ - BaseLogic(val); - return *this; -} - -PrefixedOutStream& PrefixedOutStream::operator<<(const char* str) -{ - BaseLogic(str); - return *this; -} - -PrefixedOutStream& PrefixedOutStream::operator<<(std::string& str) -{ - BaseLogic(str); - return *this; -} - -PrefixedOutStream& PrefixedOutStream::operator<<(std::streambuf* sb) -{ - BaseLogic(sb); - return *this; -} - -PrefixedOutStream& PrefixedOutStream::operator<<( - std::ostream& (*pf)(std::ostream&)) -{ - BaseLogic(pf); - return *this; -} - -PrefixedOutStream& PrefixedOutStream::operator<<(std::ios& (*pf)(std::ios&)) -{ - BaseLogic(pf); - return *this; -} - -PrefixedOutStream& PrefixedOutStream::operator<<( - std::ios_base& (*pf) (std::ios_base&)) -{ - BaseLogic(pf); - return *this; -} diff --git a/src/mlpack/core/util/prefixedoutstream_impl.hpp b/src/mlpack/core/util/prefixedoutstream_impl.hpp index 3cb9eea353..bce0b0f4cd 100644 --- a/src/mlpack/core/util/prefixedoutstream_impl.hpp +++ b/src/mlpack/core/util/prefixedoutstream_impl.hpp @@ -33,6 +33,116 @@ PrefixedOutStream& PrefixedOutStream::operator<<(const T& s) return *this; } +/** + * These are all necessary because gcc's template mechanism does not seem smart + * enough to figure out what I want to pass into operator<< without these. That + * may not be the actual case, but it works when these is here. + */ + +inline PrefixedOutStream& PrefixedOutStream::operator<<(bool val) +{ + BaseLogic(val); + return *this; +} + +inline PrefixedOutStream& PrefixedOutStream::operator<<(short val) +{ + BaseLogic(val); + return *this; +} + +inline PrefixedOutStream& PrefixedOutStream::operator<<(unsigned short val) +{ + BaseLogic(val); + return *this; +} + +inline PrefixedOutStream& PrefixedOutStream::operator<<(int val) +{ + BaseLogic(val); + return *this; +} + +inline PrefixedOutStream& PrefixedOutStream::operator<<(unsigned int val) +{ + BaseLogic(val); + return *this; +} + +inline PrefixedOutStream& PrefixedOutStream::operator<<(long val) +{ + BaseLogic(val); + return *this; +} + +inline PrefixedOutStream& PrefixedOutStream::operator<<(unsigned long val) +{ + BaseLogic(val); + return *this; +} + +inline PrefixedOutStream& PrefixedOutStream::operator<<(float val) +{ + BaseLogic(val); + return *this; +} + +inline PrefixedOutStream& PrefixedOutStream::operator<<(double val) +{ + BaseLogic(val); + return *this; +} + +inline PrefixedOutStream& PrefixedOutStream::operator<<(long double val) +{ + BaseLogic(val); + return *this; +} + +inline PrefixedOutStream& PrefixedOutStream::operator<<(void* val) +{ + BaseLogic(val); + return *this; +} + +inline PrefixedOutStream& PrefixedOutStream::operator<<(const char* str) +{ + BaseLogic(str); + return *this; +} + +inline PrefixedOutStream& PrefixedOutStream::operator<<(std::string& str) +{ + BaseLogic(str); + return *this; +} + +inline PrefixedOutStream& PrefixedOutStream::operator<<(std::streambuf* sb) +{ + BaseLogic(sb); + return *this; +} + +inline PrefixedOutStream& PrefixedOutStream::operator<<( + std::ostream& (*pf)(std::ostream&)) +{ + BaseLogic(pf); + return *this; +} + +inline PrefixedOutStream& PrefixedOutStream::operator<<(std::ios& (*pf)(std::ios&)) +{ + BaseLogic(pf); + return *this; +} + +inline PrefixedOutStream& PrefixedOutStream::operator<<( + std::ios_base& (*pf) (std::ios_base&)) +{ + BaseLogic(pf); + return *this; +} + // For non-Armadillo types. template typename std::enable_if::value>::type From 9208a60600514d9fb8b64eaba93222912ea52296 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 20 Jun 2022 16:55:07 -0400 Subject: [PATCH 11/39] Make program_doc.cpp header-only. (This required a little include juggling.) --- src/mlpack/core/util/CMakeLists.txt | 2 +- src/mlpack/core/util/binding_details.hpp | 1 - src/mlpack/core/util/forward.hpp | 2 + src/mlpack/core/util/io.hpp | 4 +- src/mlpack/core/util/program_doc.hpp | 3 ++ .../{program_doc.cpp => program_doc_impl.hpp} | 37 +++++++++++-------- 6 files changed, 30 insertions(+), 19 deletions(-) rename src/mlpack/core/util/{program_doc.cpp => program_doc_impl.hpp} (75%) diff --git a/src/mlpack/core/util/CMakeLists.txt b/src/mlpack/core/util/CMakeLists.txt index 87a2786af1..7587980ab2 100644 --- a/src/mlpack/core/util/CMakeLists.txt +++ b/src/mlpack/core/util/CMakeLists.txt @@ -26,7 +26,7 @@ set(SOURCES prefixedoutstream.hpp prefixedoutstream_impl.hpp program_doc.hpp - program_doc.cpp + program_doc_impl.hpp size_checks.hpp sfinae_utility.hpp singletons.cpp diff --git a/src/mlpack/core/util/binding_details.hpp b/src/mlpack/core/util/binding_details.hpp index 2320aed8cb..7e0b428d90 100644 --- a/src/mlpack/core/util/binding_details.hpp +++ b/src/mlpack/core/util/binding_details.hpp @@ -13,7 +13,6 @@ #define MLPACK_CORE_UTIL_BINDING_DETAILS_HPP #include -#include "program_doc.hpp" namespace mlpack { namespace util { diff --git a/src/mlpack/core/util/forward.hpp b/src/mlpack/core/util/forward.hpp index 46200721bb..f20f44ceb2 100644 --- a/src/mlpack/core/util/forward.hpp +++ b/src/mlpack/core/util/forward.hpp @@ -11,6 +11,8 @@ // Required forward declarations. namespace mlpack { +class IO; + namespace util { class Params; diff --git a/src/mlpack/core/util/io.hpp b/src/mlpack/core/util/io.hpp index 0b45563b6c..90421f3868 100644 --- a/src/mlpack/core/util/io.hpp +++ b/src/mlpack/core/util/io.hpp @@ -22,7 +22,6 @@ #include "timers.hpp" #include "binding_details.hpp" -#include "program_doc.hpp" #include "version.hpp" #include "param_data.hpp" @@ -305,4 +304,7 @@ class IO } // namespace mlpack +// This file must be included after IO is declared and fully defined. +#include "program_doc.hpp" + #endif diff --git a/src/mlpack/core/util/program_doc.hpp b/src/mlpack/core/util/program_doc.hpp index 7d6f64d7c4..5dc21826f5 100644 --- a/src/mlpack/core/util/program_doc.hpp +++ b/src/mlpack/core/util/program_doc.hpp @@ -97,4 +97,7 @@ class SeeAlso } // namespace util } // namespace mlpack +// Include implementation. +#include "program_doc_impl.hpp" + #endif diff --git a/src/mlpack/core/util/program_doc.cpp b/src/mlpack/core/util/program_doc_impl.hpp similarity index 75% rename from src/mlpack/core/util/program_doc.cpp rename to src/mlpack/core/util/program_doc_impl.hpp index f95f560c96..002477a592 100644 --- a/src/mlpack/core/util/program_doc.cpp +++ b/src/mlpack/core/util/program_doc_impl.hpp @@ -1,5 +1,5 @@ /** - * @file core/util/program_doc.cpp + * @file core/util/program_doc_impl.hpp * @author Yashwant Singh Parihar * @author Ryan Curtin * @@ -11,14 +11,14 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ +#ifndef MLPACK_CORE_UTIL_PROGRAM_DOC_IMPL_HPP +#define MLPACK_CORE_UTIL_PROGRAM_DOC_IMPL_HPP + #include "io.hpp" #include "program_doc.hpp" -#include - -using namespace mlpack; -using namespace mlpack::util; -using namespace std; +namespace mlpack { +namespace util { /** * Construct a BindingName object. When constructed, it will register itself @@ -28,8 +28,8 @@ using namespace std; * @param bindingName Name of the binding. * @param name Name displayed to user of the binding. */ -BindingName::BindingName(const std::string& bindingName, - const std::string& name) +inline BindingName::BindingName(const std::string& bindingName, + const std::string& name) { // Register this with IO. IO::AddBindingName(bindingName, name); @@ -44,8 +44,8 @@ BindingName::BindingName(const std::string& bindingName, * @param shortDescription A short two-sentence description of the binding, * what it does, and what it is useful for. */ -ShortDescription::ShortDescription(const std::string& bindingName, - const std::string& shortDescription) +inline ShortDescription::ShortDescription(const std::string& bindingName, + const std::string& shortDescription) { // Register this with IO. IO::AddShortDescription(bindingName, shortDescription); @@ -61,7 +61,7 @@ ShortDescription::ShortDescription(const std::string& bindingName, * what it is. No newline characters are necessary; this is * taken care of by IO later. */ -LongDescription::LongDescription( +inline LongDescription::LongDescription( const std::string& bindingName, const std::function& longDescription) { @@ -76,8 +76,8 @@ LongDescription::LongDescription( * @param bindingName Name of the binding. * @param example Documentation on how to use the binding. */ -Example::Example(const std::string& bindingName, - const std::function& example) +inline Example::Example(const std::string& bindingName, + const std::function& example) { // Register this with IO. IO::AddExample(bindingName, example); @@ -91,10 +91,15 @@ Example::Example(const std::string& bindingName, * @param description Description of SeeAlso. * @param link Link of SeeAlso. */ -SeeAlso::SeeAlso(const std::string& bindingName, - const std::string& description, - const std::string& link) +inline SeeAlso::SeeAlso(const std::string& bindingName, + const std::string& description, + const std::string& link) { // Register this with IO. IO::AddSeeAlso(bindingName, description, link); } + +} // namespace util +} // namespace mlpack + +#endif From 6c6a50b1bc8cfd75197f565ac79c6108a5a79c1f Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 20 Jun 2022 21:26:06 -0400 Subject: [PATCH 12/39] Adapt log.cpp and io.cpp. --- src/mlpack/core/util/CMakeLists.txt | 4 +- src/mlpack/core/util/io.hpp | 3 ++ src/mlpack/core/util/{io.cpp => io_impl.hpp} | 53 +++++++++++-------- src/mlpack/core/util/log.hpp | 3 ++ .../core/util/{log.cpp => log_impl.hpp} | 16 ++++-- 5 files changed, 49 insertions(+), 30 deletions(-) rename src/mlpack/core/util/{io.cpp => io_impl.hpp} (79%) rename src/mlpack/core/util/{log.cpp => log_impl.hpp} (70%) diff --git a/src/mlpack/core/util/CMakeLists.txt b/src/mlpack/core/util/CMakeLists.txt index 7587980ab2..8357a353bc 100644 --- a/src/mlpack/core/util/CMakeLists.txt +++ b/src/mlpack/core/util/CMakeLists.txt @@ -9,12 +9,12 @@ set(SOURCES binding_details.hpp forward.hpp io.hpp - io.cpp + io_impl.hpp deprecated.hpp hyphenate_string.hpp is_std_vector.hpp log.hpp - log.cpp + log_impl.hpp mlpack_main.hpp nulloutstream.hpp param.hpp diff --git a/src/mlpack/core/util/io.hpp b/src/mlpack/core/util/io.hpp index 90421f3868..131ab12e61 100644 --- a/src/mlpack/core/util/io.hpp +++ b/src/mlpack/core/util/io.hpp @@ -307,4 +307,7 @@ class IO // This file must be included after IO is declared and fully defined. #include "program_doc.hpp" +// Include the implementation. +#include "io_impl.hpp" + #endif diff --git a/src/mlpack/core/util/io.cpp b/src/mlpack/core/util/io_impl.hpp similarity index 79% rename from src/mlpack/core/util/io.cpp rename to src/mlpack/core/util/io_impl.hpp index cb2c39625d..798f34d65b 100644 --- a/src/mlpack/core/util/io.cpp +++ b/src/mlpack/core/util/io_impl.hpp @@ -1,5 +1,5 @@ /** - * @file core/util/io.cpp + * @file core/util/io_impl.hpp * @author Matthew Amidon * * Implementation of the IO module for parsing parameters. @@ -9,31 +9,33 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ +#ifndef MLPACK_CORE_UTIL_IO_IMPL_HPP +#define MLPACK_CORE_UTIL_IO_IMPL_HPP #include "io.hpp" #include "log.hpp" #include "hyphenate_string.hpp" -using namespace mlpack; -using namespace mlpack::util; +namespace mlpack { /* Constructors, Destructors, Copy */ /* Make the constructor private, to preclude unauthorized instances */ -IO::IO() +inline IO::IO() { return; } // Private copy constructor; don't want copies floating around. -IO::IO(const IO& /* other */) +inline IO::IO(const IO& /* other */) { return; } // Private copy operator; don't want copies floating around. -IO& IO::operator=(const IO& /* other */) { return *this; } +inline IO& IO::operator=(const IO& /* other */) { return *this; } -void IO::AddParameter(const std::string& bindingName, ParamData&& data) +inline void IO::AddParameter(const std::string& bindingName, + util::ParamData&& data) { // Temporarily define color code escape sequences. #ifndef _WIN32 @@ -93,9 +95,9 @@ void IO::AddParameter(const std::string& bindingName, ParamData&& data) * @param name Name of the function. * @param func Function to call. */ -void IO::AddFunction(const std::string& type, - const std::string& name, - void (*func)(util::ParamData&, const void*, void*)) +inline void IO::AddFunction(const std::string& type, + const std::string& name, + void (*func)(util::ParamData&, const void*, void*)) { std::lock_guard lock(GetSingleton().mapMutex); GetSingleton().functionMap[type][name] = func; @@ -107,7 +109,8 @@ void IO::AddFunction(const std::string& type, * @param bindingName Name of the binding to add the user-friendly name for. * @param name User-friendly name. */ -void IO::AddBindingName(const std::string& bindingName, const std::string& name) +inline void IO::AddBindingName(const std::string& bindingName, + const std::string& name) { std::lock_guard lock(GetSingleton().mapMutex); GetSingleton().docs[bindingName].name = name; @@ -119,8 +122,8 @@ void IO::AddBindingName(const std::string& bindingName, const std::string& name) * @param bindingName Name of the binding to add the description for. * @param shortDescription Description to use. */ -void IO::AddShortDescription(const std::string& bindingName, - const std::string& shortDescription) +inline void IO::AddShortDescription(const std::string& bindingName, + const std::string& shortDescription) { std::lock_guard lock(GetSingleton().docMutex); GetSingleton().docs[bindingName].shortDescription = shortDescription; @@ -132,7 +135,7 @@ void IO::AddShortDescription(const std::string& bindingName, * @param bindingName Name of the binding to add the description for. * @param longDescription Function that returns the long description. */ -void IO::AddLongDescription( +inline void IO::AddLongDescription( const std::string& bindingName, const std::function& longDescription) { @@ -146,8 +149,8 @@ void IO::AddLongDescription( * @param bindingName Name of the binding to add the example for. * @param example Function that returns the example. */ -void IO::AddExample(const std::string& bindingName, - const std::function& example) +inline void IO::AddExample(const std::string& bindingName, + const std::function& example) { std::lock_guard lock(GetSingleton().docMutex); GetSingleton().docs[bindingName].example.push_back(std::move(example)); @@ -160,9 +163,9 @@ void IO::AddExample(const std::string& bindingName, * @param description Description of the SeeAlso. * @param link Link of the SeeAlso. */ -void IO::AddSeeAlso(const std::string& bindingName, - const std::string& description, - const std::string& link) +inline void IO::AddSeeAlso(const std::string& bindingName, + const std::string& description, + const std::string& link) { std::lock_guard lock(GetSingleton().docMutex); GetSingleton().docs[bindingName].seeAlso.push_back( @@ -170,14 +173,14 @@ void IO::AddSeeAlso(const std::string& bindingName, } // Returns the sole instance of this class. -IO& IO::GetSingleton() +inline IO& IO::GetSingleton() { static IO singleton; return singleton; } // Returns the sole instance of the timers. -util::Timers& IO::GetTimers() +inline util::Timers& IO::GetTimers() { return GetSingleton().timer; } @@ -187,7 +190,7 @@ util::Timers& IO::GetTimers() * binding `bindingName`. This is intended to be called at the beginning of * the run of a binding. */ -util::Params IO::Parameters(const std::string& bindingName) +inline util::Params IO::Parameters(const std::string& bindingName) { // We don't need a mutex here, because we are only randomly accessing elements // of the maps. @@ -205,6 +208,10 @@ util::Params IO::Parameters(const std::string& bindingName) GetSingleton().parameters[""]; resultParams.insert(persistentParams.begin(), persistentParams.end()); - return Params(resultAliases, resultParams, GetSingleton().functionMap, + return util::Params(resultAliases, resultParams, GetSingleton().functionMap, bindingName, GetSingleton().docs[bindingName]); } + +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/util/log.hpp b/src/mlpack/core/util/log.hpp index 351e362c88..33bba0d626 100644 --- a/src/mlpack/core/util/log.hpp +++ b/src/mlpack/core/util/log.hpp @@ -95,4 +95,7 @@ class Log }; // namespace mlpack +// Include implementation. +#include "log_impl.hpp" + #endif diff --git a/src/mlpack/core/util/log.cpp b/src/mlpack/core/util/log_impl.hpp similarity index 70% rename from src/mlpack/core/util/log.cpp rename to src/mlpack/core/util/log_impl.hpp index d8b2767025..e55e7f7d8c 100644 --- a/src/mlpack/core/util/log.cpp +++ b/src/mlpack/core/util/log_impl.hpp @@ -1,5 +1,5 @@ /** - * @file core/util/log.cpp + * @file core/util/log_impl.hpp * @author Matthew Amidon * * Implementation of the Log class. @@ -9,18 +9,20 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ +#ifndef MLPACK_CORE_UTIL_LOG_IMPL_HPP +#define MLPACK_CORE_UTIL_LOG_IMPL_HPP + #include "log.hpp" #ifdef HAS_BFD_DL #include "backtrace.hpp" #endif -using namespace mlpack; -using namespace mlpack::util; +namespace mlpack { // Only do anything for Assert() if in debugging mode. #ifdef DEBUG -void Log::Assert(bool condition, const std::string& message) +inline void Log::Assert(bool condition, const std::string& message) { if (!condition) { @@ -35,6 +37,10 @@ void Log::Assert(bool condition, const std::string& message) } } #else -void Log::Assert(bool /* condition */, const std::string& /* message */) +inline void Log::Assert(bool /* condition */, const std::string& /* message */) { } #endif + +} // namespace mlpack + +#endif From 1f732a17deba11462be112fafcb6206a7e7f89ad Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 21 Jun 2022 17:54:02 -0400 Subject: [PATCH 13/39] Adapt a last few files to header-only. --- src/mlpack/bindings/tests/CMakeLists.txt | 10 +-- src/mlpack/core/util/CMakeLists.txt | 1 - src/mlpack/core/util/log.hpp | 81 +++++++++++-------- .../util/{singletons.cpp => singletons.hpp} | 24 +++--- src/mlpack/tests/CMakeLists.txt | 1 + 5 files changed, 70 insertions(+), 47 deletions(-) rename src/mlpack/core/util/{singletons.cpp => singletons.hpp} (64%) diff --git a/src/mlpack/bindings/tests/CMakeLists.txt b/src/mlpack/bindings/tests/CMakeLists.txt index 7e87aa4bfc..6845830e49 100644 --- a/src/mlpack/bindings/tests/CMakeLists.txt +++ b/src/mlpack/bindings/tests/CMakeLists.txt @@ -1,6 +1,6 @@ -# Define the files we need to compile. -# Anything not in this list will not be compiled into mlpack. -set(SOURCES +# Define the files we need to compile into the test executable. +# Anything not in this list will not be compiled into mlpack_test. +set(BINDING_SOURCES clean_memory.hpp clean_memory.cpp test_option.hpp @@ -17,9 +17,9 @@ set(SOURCES # Add directory name to sources. set(DIR_SRCS) -foreach(file ${SOURCES}) +foreach(file ${BINDING_SOURCES}) set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) endforeach() # Append source (with directory name) to list of all mlpack sources (used at the # parent scope). -set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) +set(MLPACK_TEST_SRCS ${MLPACK_TEST_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/core/util/CMakeLists.txt b/src/mlpack/core/util/CMakeLists.txt index 8357a353bc..1cc1dfa9b9 100644 --- a/src/mlpack/core/util/CMakeLists.txt +++ b/src/mlpack/core/util/CMakeLists.txt @@ -29,7 +29,6 @@ set(SOURCES program_doc_impl.hpp size_checks.hpp sfinae_utility.hpp - singletons.cpp timers.hpp timers_impl.hpp to_lower.hpp diff --git a/src/mlpack/core/util/log.hpp b/src/mlpack/core/util/log.hpp index 33bba0d626..78f4725bf6 100644 --- a/src/mlpack/core/util/log.hpp +++ b/src/mlpack/core/util/log.hpp @@ -53,47 +53,64 @@ namespace mlpack { * * @see PrefixedOutStream, NullOutStream, IO */ -class Log -{ - public: - /** - * Checks if the specified condition is true. - * If not, halts program execution and prints a custom error message. - * Does nothing in non-debug mode. - */ - static void Assert(bool condition, - const std::string& message = "Assert Failed."); +namespace Log { - /** - * MLPACK_EXPORT is required for global variables, so that they are properly - * exported by the Windows compiler. - */ +/** + * Checks if the specified condition is true. + * If not, halts program execution and prints a custom error message. + * Does nothing in non-debug mode. + */ +void Assert(bool condition, + const std::string& message = "Assert Failed."); - // We only use PrefixedOutStream if the program is compiled with debug - // symbols. -#ifdef DEBUG - //! Prints debug output with the appropriate tag: [DEBUG]. - static MLPACK_EXPORT util::PrefixedOutStream Debug; +/** + * MLPACK_EXPORT is required for global variables, so that they are properly + * exported by the Windows compiler. + */ + +// Color code escape sequences -- but not on Windows. +#ifndef _WIN32 + #define BASH_RED "\033[0;31m" + #define BASH_GREEN "\033[0;32m" + #define BASH_YELLOW "\033[0;33m" + #define BASH_CYAN "\033[0;36m" + #define BASH_CLEAR "\033[0m" #else - //! Dumps debug output into the bit nether regions. - static MLPACK_EXPORT util::NullOutStream Debug; + #define BASH_RED "" + #define BASH_GREEN "" + #define BASH_YELLOW "" + #define BASH_CYAN "" + #define BASH_CLEAR "" #endif - //! Prints informational messages if --verbose is specified, prefixed with - //! [INFO ]. - static MLPACK_EXPORT util::PrefixedOutStream Info; +#ifdef DEBUG +static util::PrefixedOutStream Debug = + util::PrefixedOutStream(MLPACK_COUT_STREAM, + BASH_CYAN "[DEBUG] " BASH_CLEAR); +#else +static util::NullOutStream Debug = util::NullOutStream(); +#endif - //! Prints warning messages prefixed with [WARN ]. - static MLPACK_EXPORT util::PrefixedOutStream Warn; +static util::PrefixedOutStream Info = + util::PrefixedOutStream(MLPACK_COUT_STREAM, + BASH_GREEN "[INFO ] " BASH_CLEAR, + true /* unless --verbose */, + false); - //! Prints fatal messages prefixed with [FATAL], then terminates the program. - static MLPACK_EXPORT util::PrefixedOutStream Fatal; +static util::PrefixedOutStream Warn = + util::PrefixedOutStream(MLPACK_COUT_STREAM, + BASH_YELLOW "[WARN ] " BASH_CLEAR, + false, + false); - //! Reference to cout, if necessary. - static std::ostream& cout; -}; +static util::PrefixedOutStream Fatal = + util::PrefixedOutStream(MLPACK_CERR_STREAM, + BASH_RED "[FATAL] " BASH_CLEAR, + false, + true /* fatal */); -}; // namespace mlpack +} // namespace Log +} // namespace mlpack // Include implementation. #include "log_impl.hpp" diff --git a/src/mlpack/core/util/singletons.cpp b/src/mlpack/core/util/singletons.hpp similarity index 64% rename from src/mlpack/core/util/singletons.cpp rename to src/mlpack/core/util/singletons.hpp index 2e653ef3e8..ca951c42dc 100644 --- a/src/mlpack/core/util/singletons.cpp +++ b/src/mlpack/core/util/singletons.hpp @@ -1,5 +1,5 @@ /** - * @file core/util/singletons.cpp + * @file core/util/singletons.hpp * @author Ryan Curtin * * Declaration of singletons in libmlpack.so. @@ -9,13 +9,13 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ +#ifndef MLPACK_CORE_UTIL_SINGLETONS_HPP +#define MLPACK_CORE_UTIL_SINGLETONS_HPP + #include "io.hpp" #include "log.hpp" #include -using namespace mlpack; -using namespace mlpack::util; - // Color code escape sequences -- but not on Windows. #ifndef _WIN32 #define BASH_RED "\033[0;31m" @@ -31,16 +31,22 @@ using namespace mlpack::util; #define BASH_CLEAR "" #endif +namespace mlpack { + #ifdef DEBUG -PrefixedOutStream Log::Debug = PrefixedOutStream(MLPACK_COUT_STREAM, +static util::PrefixedOutStream Log::Debug = util::PrefixedOutStream(MLPACK_COUT_STREAM, BASH_CYAN "[DEBUG] " BASH_CLEAR); #else -NullOutStream Log::Debug = NullOutStream(); +static util::NullOutStream Log::Debug = util::NullOutStream(); #endif -PrefixedOutStream Log::Info = PrefixedOutStream(MLPACK_COUT_STREAM, +static util::PrefixedOutStream Log::Info = util::PrefixedOutStream(MLPACK_COUT_STREAM, BASH_GREEN "[INFO ] " BASH_CLEAR, true /* unless --verbose */, false); -PrefixedOutStream Log::Warn = PrefixedOutStream(MLPACK_COUT_STREAM, +static util::PrefixedOutStream Log::Warn = util::PrefixedOutStream(MLPACK_COUT_STREAM, BASH_YELLOW "[WARN ] " BASH_CLEAR, false, false); -PrefixedOutStream Log::Fatal = PrefixedOutStream(MLPACK_CERR_STREAM, +static util::PrefixedOutStream Log::Fatal = util::PrefixedOutStream(MLPACK_CERR_STREAM, BASH_RED "[FATAL] " BASH_CLEAR, false, true /* fatal */); + +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 6e6b0840b5..2344401810 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -3,6 +3,7 @@ include(CTest) # mlpack test executable. add_executable(mlpack_test EXCLUDE_FROM_ALL + ${MLPACK_TEST_SRCS} activation_functions_test.cpp adaboost_test.cpp akfn_test.cpp From 1d69d83910e81160246ecf243e75c358d1193b2a Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 21 Jun 2022 17:59:55 -0400 Subject: [PATCH 14/39] Remove unused function: it turned out to be unnecessary. --- src/mlpack/core/data/dataset_mapper_impl.hpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/mlpack/core/data/dataset_mapper_impl.hpp b/src/mlpack/core/data/dataset_mapper_impl.hpp index e5574b58e2..7422adc673 100644 --- a/src/mlpack/core/data/dataset_mapper_impl.hpp +++ b/src/mlpack/core/data/dataset_mapper_impl.hpp @@ -248,12 +248,6 @@ inline void DatasetMapper::Policy(PolicyType&& policy) this->policy = std::forward(policy); } -// Implementation of utility function needed by Params class. -inline arma::mat& GetMatrix(std::tuple& t) -{ - return std::get<1>(t); -} - } // namespace data } // namespace mlpack From 6a1d77cac2191d27b733656b79c2bfc04b4e32cd Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 21 Jun 2022 18:01:29 -0400 Subject: [PATCH 15/39] singletons.hpp is now inlined in log.hpp. --- src/mlpack/core/util/singletons.hpp | 52 ----------------------------- 1 file changed, 52 deletions(-) delete mode 100644 src/mlpack/core/util/singletons.hpp diff --git a/src/mlpack/core/util/singletons.hpp b/src/mlpack/core/util/singletons.hpp deleted file mode 100644 index ca951c42dc..0000000000 --- a/src/mlpack/core/util/singletons.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/** - * @file core/util/singletons.hpp - * @author Ryan Curtin - * - * Declaration of singletons in libmlpack.so. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_CORE_UTIL_SINGLETONS_HPP -#define MLPACK_CORE_UTIL_SINGLETONS_HPP - -#include "io.hpp" -#include "log.hpp" -#include - -// Color code escape sequences -- but not on Windows. -#ifndef _WIN32 - #define BASH_RED "\033[0;31m" - #define BASH_GREEN "\033[0;32m" - #define BASH_YELLOW "\033[0;33m" - #define BASH_CYAN "\033[0;36m" - #define BASH_CLEAR "\033[0m" -#else - #define BASH_RED "" - #define BASH_GREEN "" - #define BASH_YELLOW "" - #define BASH_CYAN "" - #define BASH_CLEAR "" -#endif - -namespace mlpack { - -#ifdef DEBUG -static util::PrefixedOutStream Log::Debug = util::PrefixedOutStream(MLPACK_COUT_STREAM, - BASH_CYAN "[DEBUG] " BASH_CLEAR); -#else -static util::NullOutStream Log::Debug = util::NullOutStream(); -#endif - -static util::PrefixedOutStream Log::Info = util::PrefixedOutStream(MLPACK_COUT_STREAM, - BASH_GREEN "[INFO ] " BASH_CLEAR, true /* unless --verbose */, false); -static util::PrefixedOutStream Log::Warn = util::PrefixedOutStream(MLPACK_COUT_STREAM, - BASH_YELLOW "[WARN ] " BASH_CLEAR, false, false); -static util::PrefixedOutStream Log::Fatal = util::PrefixedOutStream(MLPACK_CERR_STREAM, - BASH_RED "[FATAL] " BASH_CLEAR, false, true /* fatal */); - -} // namespace mlpack - -#endif From 0af56d2dbf940a51b689e15d835c3b4098388716 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 21 Jun 2022 22:45:17 -0400 Subject: [PATCH 16/39] Propagate MLPACK_TEST_SRCS to the parent directory. --- src/mlpack/bindings/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/bindings/CMakeLists.txt b/src/mlpack/bindings/CMakeLists.txt index c9c96328e5..ad759d5594 100644 --- a/src/mlpack/bindings/CMakeLists.txt +++ b/src/mlpack/bindings/CMakeLists.txt @@ -15,6 +15,7 @@ endforeach() set(MARKDOWN_CATEGORIES ${MARKDOWN_CATEGORIES} PARENT_SCOPE) set(MLPACK_SRCS ${MLPACK_SRCS} PARENT_SCOPE) +set(MLPACK_TEST_SRCS ${MLPACK_TEST_SRCS} PARENT_SCOPE) set(MLPACK_PYXS ${MLPACK_PYXS} PARENT_SCOPE) set(DISABLE_CFLAGS ${DISABLE_CFLAGS} PARENT_SCOPE) set(BUILDING_PYTHON_BINDINGS ${BUILDING_PYTHON_BINDINGS} PARENT_SCOPE) From 107a985f72e4762ff21c91a046dfe46889978d65 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 23 Jun 2022 14:39:01 -0400 Subject: [PATCH 17/39] Adapt include ordering for header-only. --- src/mlpack/base.hpp | 124 +++++++++++++++++++++ src/mlpack/core/util/prefixedoutstream.hpp | 2 +- src/mlpack/prereqs.hpp | 118 +------------------- 3 files changed, 128 insertions(+), 116 deletions(-) create mode 100644 src/mlpack/base.hpp diff --git a/src/mlpack/base.hpp b/src/mlpack/base.hpp new file mode 100644 index 0000000000..05aae68577 --- /dev/null +++ b/src/mlpack/base.hpp @@ -0,0 +1,124 @@ +/** + * @file base.hpp + * + * The most basic core includes that mlpack expects; standard C++ includes and + * Armadillo only + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_BASE_HPP +#define MLPACK_BASE_HPP + +// First, check if Armadillo was included before, warning if so. +#ifdef ARMA_INCLUDES +#pragma message "Armadillo was included before mlpack; this can sometimes cause\ + problems. It should only be necessary to include and not \ +." +#endif + +// Defining _USE_MATH_DEFINES should set M_PI. +#define _USE_MATH_DEFINES +#include + +// Next, standard includes. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// But if it's not defined, we'll do it. +#ifndef M_PI + #define M_PI 3.141592653589793238462643383279 +#endif + +// MLPACK_COUT_STREAM is used to change the default stream for printing +// purpose. +#if !defined(MLPACK_COUT_STREAM) + #define MLPACK_COUT_STREAM std::cout +#endif + +// MLPACK_CERR_STREAM is used to change the stream for printing warnings +// and errors. +#if !defined(MLPACK_CERR_STREAM) + #define MLPACK_CERR_STREAM std::cerr +#endif + +// Give ourselves a nice way to force functions to be inline if we need. +#define force_inline +#if defined(__GNUG__) && !defined(DEBUG) + #undef force_inline + #define force_inline __attribute__((always_inline)) +#elif defined(_MSC_VER) && !defined(DEBUG) + #undef force_inline + #define force_inline __forceinline +#endif + +// Backport this functionality from C++14, if it doesn't exist. +#if __cplusplus <= 201103L +#if !defined(_MSC_VER) || _MSC_VER <= 1800 +namespace std { + +template +using enable_if_t = typename enable_if::type; + +} +#endif +#endif + +// Backport std::any from C+17 to C++11 to replace boost::any. +// Use mnmlstc backport implementation only if compiler does not +// support C++17. +#if __cplusplus < 201703L + #include + #include + #define ANY core::v2::any + #define ANY_CAST core::v2::any_cast + #define STRING_VIEW core::v2::string_view +#else + #include + #include + #define ANY std::any + #define ANY_CAST std::any_cast + #define STRING_VIEW std::string_view +#endif + +// Now include Armadillo through the special mlpack extensions. +#include +#include + +// On Visual Studio, disable C4519 (default arguments for function templates) +// since it's by default an error, which doesn't even make any sense because +// it's part of the C++11 standard. +#ifdef _MSC_VER + #pragma warning(disable : 4519) + #define ARMA_USE_CXX11 +#endif + +// Ensure that the user isn't doing something stupid with their Armadillo +// defines. +#include + +// This can be removed with Visual Studio supports an OpenMP version with +// unsigned loop variables. +#ifdef _WIN32 + #define omp_size_t intmax_t +#else + #define omp_size_t size_t +#endif + +// We need to be able to mark functions deprecated. +#include + +#endif diff --git a/src/mlpack/core/util/prefixedoutstream.hpp b/src/mlpack/core/util/prefixedoutstream.hpp index 664c3b6203..fddf90439f 100644 --- a/src/mlpack/core/util/prefixedoutstream.hpp +++ b/src/mlpack/core/util/prefixedoutstream.hpp @@ -13,7 +13,7 @@ #ifndef MLPACK_CORE_UTIL_PREFIXEDOUTSTREAM_HPP #define MLPACK_CORE_UTIL_PREFIXEDOUTSTREAM_HPP -#include +#include namespace mlpack { namespace util { diff --git a/src/mlpack/prereqs.hpp b/src/mlpack/prereqs.hpp index 6310364f55..c71fcc9f22 100644 --- a/src/mlpack/prereqs.hpp +++ b/src/mlpack/prereqs.hpp @@ -1,7 +1,8 @@ /** * @file prereqs.hpp * - * The core includes that mlpack expects; standard C++ includes and Armadillo. + * The core includes that mlpack expects; standard C++ includes, Armadillo, + * cereal, and a few basic mlpack utilities. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -11,97 +12,7 @@ #ifndef MLPACK_PREREQS_HPP #define MLPACK_PREREQS_HPP -// First, check if Armadillo was included before, warning if so. -#ifdef ARMA_INCLUDES -#pragma message "Armadillo was included before mlpack; this can sometimes cause\ - problems. It should only be necessary to include and not \ -." -#endif - -// Defining _USE_MATH_DEFINES should set M_PI. -#define _USE_MATH_DEFINES -#include - -// Next, standard includes. -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// But if it's not defined, we'll do it. -#ifndef M_PI - #define M_PI 3.141592653589793238462643383279 -#endif - -// MLPACK_COUT_STREAM is used to change the default stream for printing -// purpose. -#if !defined(MLPACK_COUT_STREAM) - #define MLPACK_COUT_STREAM std::cout -#endif - -// MLPACK_CERR_STREAM is used to change the stream for printing warnings -// and errors. -#if !defined(MLPACK_CERR_STREAM) - #define MLPACK_CERR_STREAM std::cerr -#endif - -// Give ourselves a nice way to force functions to be inline if we need. -#define force_inline -#if defined(__GNUG__) && !defined(DEBUG) - #undef force_inline - #define force_inline __attribute__((always_inline)) -#elif defined(_MSC_VER) && !defined(DEBUG) - #undef force_inline - #define force_inline __forceinline -#endif - -// Backport this functionality from C++14, if it doesn't exist. -#if __cplusplus <= 201103L -#if !defined(_MSC_VER) || _MSC_VER <= 1800 -namespace std { - -template -using enable_if_t = typename enable_if::type; - -} -#endif -#endif - -// Backport std::any from C+17 to C++11 to replace boost::any. -// Use mnmlstc backport implementation only if compiler does not -// support C++17. -#if __cplusplus < 201703L - #include - #include - #define ANY core::v2::any - #define ANY_CAST core::v2::any_cast - #define STRING_VIEW core::v2::string_view -#else - #include - #include - #define ANY std::any - #define ANY_CAST std::any_cast - #define STRING_VIEW std::string_view -#endif - -// Increase the number of template arguments for the boost list class. -#undef BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS -#undef BOOST_MPL_LIMIT_LIST_SIZE -#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS -#define BOOST_MPL_LIMIT_LIST_SIZE 50 - -// Now include Armadillo through the special mlpack extensions. -#include -#include +#include "base.hpp" #include #include @@ -122,33 +33,10 @@ using enable_if_t = typename enable_if::type; #include #include -// On Visual Studio, disable C4519 (default arguments for function templates) -// since it's by default an error, which doesn't even make any sense because -// it's part of the C++11 standard. -#ifdef _MSC_VER - #pragma warning(disable : 4519) - #define ARMA_USE_CXX11 -#endif - -// Ensure that the user isn't doing something stupid with their Armadillo -// defines. -#include - // All code should have access to logging. #include #include -// This can be removed with Visual Studio supports an OpenMP version with -// unsigned loop variables. -#ifdef _WIN32 - #define omp_size_t intmax_t -#else - #define omp_size_t size_t -#endif - -// We need to be able to mark functions deprecated. -#include - // Include ready to use utility function to check sizes of datasets. #include From 45b9a9a4bdfd113c0a5b5ef49ca8e6b7e5062833 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 23 Jun 2022 15:36:30 -0400 Subject: [PATCH 18/39] Add forward declaration of IO. --- src/mlpack/core/util/timers_impl.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mlpack/core/util/timers_impl.hpp b/src/mlpack/core/util/timers_impl.hpp index 2ca43f80ca..80d40e5df5 100644 --- a/src/mlpack/core/util/timers_impl.hpp +++ b/src/mlpack/core/util/timers_impl.hpp @@ -12,6 +12,8 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include "timers.hpp" + +#include "forward.hpp" #include "io.hpp" #include "log.hpp" From b9f7c1ff8bc54c76a1bec2ace8f661c64a5e2484 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 23 Jun 2022 18:11:57 -0400 Subject: [PATCH 19/39] Header include ordering cleanup: fix IO/Timers mutual dependency. --- src/mlpack/bindings/cli/mlpack_main.hpp | 2 +- src/mlpack/core/data/load.hpp | 1 - src/mlpack/core/data/load_impl.hpp | 1 - src/mlpack/core/util/forward.hpp | 1 + src/mlpack/core/util/io.hpp | 3 +++ src/mlpack/core/util/timers.hpp | 4 ++-- .../bayesian_linear_regression/bayesian_linear_regression.hpp | 2 -- src/mlpack/methods/lars/lars.hpp | 2 -- src/mlpack/methods/radical/radical.hpp | 3 --- src/mlpack/prereqs.hpp | 2 +- 10 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/mlpack/bindings/cli/mlpack_main.hpp b/src/mlpack/bindings/cli/mlpack_main.hpp index 5e4832e675..9be224fbab 100644 --- a/src/mlpack/bindings/cli/mlpack_main.hpp +++ b/src/mlpack/bindings/cli/mlpack_main.hpp @@ -80,7 +80,7 @@ using Option = mlpack::bindings::cli::CLIOption; } #include -#include +#include #include #include diff --git a/src/mlpack/core/data/load.hpp b/src/mlpack/core/data/load.hpp index 195838affe..c237c5cc00 100644 --- a/src/mlpack/core/data/load.hpp +++ b/src/mlpack/core/data/load.hpp @@ -15,7 +15,6 @@ #define MLPACK_CORE_DATA_LOAD_HPP #include -#include #include #include "format.hpp" diff --git a/src/mlpack/core/data/load_impl.hpp b/src/mlpack/core/data/load_impl.hpp index 03993716db..5e8b7a349d 100644 --- a/src/mlpack/core/data/load_impl.hpp +++ b/src/mlpack/core/data/load_impl.hpp @@ -18,7 +18,6 @@ #include #include -#include #include "extension.hpp" #include "detect_file_type.hpp" diff --git a/src/mlpack/core/util/forward.hpp b/src/mlpack/core/util/forward.hpp index f20f44ceb2..688d67c419 100644 --- a/src/mlpack/core/util/forward.hpp +++ b/src/mlpack/core/util/forward.hpp @@ -16,6 +16,7 @@ class IO; namespace util { class Params; +class Timers; } // namespace util diff --git a/src/mlpack/core/util/io.hpp b/src/mlpack/core/util/io.hpp index 131ab12e61..a91359bb54 100644 --- a/src/mlpack/core/util/io.hpp +++ b/src/mlpack/core/util/io.hpp @@ -310,4 +310,7 @@ class IO // Include the implementation. #include "io_impl.hpp" +// Now include the implementation of the timers. +#include "timers_impl.hpp" + #endif diff --git a/src/mlpack/core/util/timers.hpp b/src/mlpack/core/util/timers.hpp index ee7279c004..86266fcef6 100644 --- a/src/mlpack/core/util/timers.hpp +++ b/src/mlpack/core/util/timers.hpp @@ -182,7 +182,7 @@ class Timers } // namespace util } // namespace mlpack -// Include implementation. -#include "timers_impl.hpp" +// Note that the implementation is not included, to avoid include ordering +// issues! #endif // MLPACK_CORE_UTILITIES_TIMERS_HPP diff --git a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp index 7665c3e9a6..417e795bff 100644 --- a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp +++ b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp @@ -16,8 +16,6 @@ #define MLPACK_METHODS_BAYESIAN_LINEAR_REGRESSION_HPP #include -#include -#include namespace mlpack { namespace regression { diff --git a/src/mlpack/methods/lars/lars.hpp b/src/mlpack/methods/lars/lars.hpp index 37dd212507..d019fec900 100644 --- a/src/mlpack/methods/lars/lars.hpp +++ b/src/mlpack/methods/lars/lars.hpp @@ -25,8 +25,6 @@ #define MLPACK_METHODS_LARS_LARS_HPP #include -#include -#include namespace mlpack { namespace regression { diff --git a/src/mlpack/methods/radical/radical.hpp b/src/mlpack/methods/radical/radical.hpp index 5d79b3daad..c482e9bf46 100644 --- a/src/mlpack/methods/radical/radical.hpp +++ b/src/mlpack/methods/radical/radical.hpp @@ -15,9 +15,6 @@ #define MLPACK_METHODS_RADICAL_RADICAL_HPP #include -#include -#include -#include namespace mlpack { namespace radical { diff --git a/src/mlpack/prereqs.hpp b/src/mlpack/prereqs.hpp index c71fcc9f22..202788013e 100644 --- a/src/mlpack/prereqs.hpp +++ b/src/mlpack/prereqs.hpp @@ -35,7 +35,7 @@ // All code should have access to logging. #include -#include +#include // Include ready to use utility function to check sizes of datasets. #include From 230972779e98eba5b325294745f16050b23b6098 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 23 Jun 2022 20:31:33 -0400 Subject: [PATCH 20/39] Add new file to R build. --- src/mlpack/bindings/R/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/bindings/R/CMakeLists.txt b/src/mlpack/bindings/R/CMakeLists.txt index bd70f206fb..fa7cfaaa1e 100644 --- a/src/mlpack/bindings/R/CMakeLists.txt +++ b/src/mlpack/bindings/R/CMakeLists.txt @@ -414,6 +414,7 @@ macro (post_r_setup) # Then copy each of the header and source files over to that directory. set(MLPACK_SOURCES "${CMAKE_BINARY_DIR}/src/mlpack/mlpack_export.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/base.hpp" "${CMAKE_CURRENT_SOURCE_DIR}/prereqs.hpp" "${CMAKE_CURRENT_SOURCE_DIR}/core.hpp" ) From 31e9236cff57ccb898f453ed69a74efa7e580279 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Jun 2022 18:25:10 -0400 Subject: [PATCH 21/39] Update src/mlpack/core/util/forward.hpp Co-authored-by: Marcus Edel --- src/mlpack/core/util/forward.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/util/forward.hpp b/src/mlpack/core/util/forward.hpp index 688d67c419..e857fdc235 100644 --- a/src/mlpack/core/util/forward.hpp +++ b/src/mlpack/core/util/forward.hpp @@ -1,5 +1,5 @@ /** - * @file forward.hpp + * @file core/util/forward.hpp * @author Ryan Curtin * * Forward declaration of components from other subdirectories necessary for From c2c898dc55473538f2741f237563ffa290cd50aa Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Jun 2022 18:25:15 -0400 Subject: [PATCH 22/39] Update src/mlpack/core/data/check_categorical_param.hpp Co-authored-by: Marcus Edel --- src/mlpack/core/data/check_categorical_param.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/data/check_categorical_param.hpp b/src/mlpack/core/data/check_categorical_param.hpp index 06085e7261..988b978878 100644 --- a/src/mlpack/core/data/check_categorical_param.hpp +++ b/src/mlpack/core/data/check_categorical_param.hpp @@ -1,5 +1,5 @@ /** - * @file check_categorical_param.hpp + * @file core/data/check_categorical_param.hpp * @author Ryan Curtin * * This file provides an implementation of a simple function to check the values From d7e705d11fb1783ec475dce7e92b154626d2a1c4 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Jun 2022 18:33:01 -0400 Subject: [PATCH 23/39] Try a few inclusion changes to make the static analysis checker happier. --- src/mlpack/core/util/forward.hpp | 2 ++ src/mlpack/core/util/param_data.hpp | 15 +-------------- src/mlpack/core/util/params.hpp | 1 + 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/mlpack/core/util/forward.hpp b/src/mlpack/core/util/forward.hpp index e857fdc235..edccc5c6e8 100644 --- a/src/mlpack/core/util/forward.hpp +++ b/src/mlpack/core/util/forward.hpp @@ -8,6 +8,8 @@ #ifndef MLPACK_CORE_UTIL_FORWARD_HPP #define MLPACK_CORE_UTIL_FORWARD_HPP +#include + // Required forward declarations. namespace mlpack { diff --git a/src/mlpack/core/util/param_data.hpp b/src/mlpack/core/util/param_data.hpp index d760f75c34..93da599c9b 100644 --- a/src/mlpack/core/util/param_data.hpp +++ b/src/mlpack/core/util/param_data.hpp @@ -13,26 +13,13 @@ #ifndef MLPACK_CORE_UTIL_PARAM_DATA_HPP #define MLPACK_CORE_UTIL_PARAM_DATA_HPP -#include +#include /** * The TYPENAME macro is used internally to convert a type into a string. */ #define TYPENAME(x) (std::string(typeid(x).name())) -namespace mlpack { -namespace data { - -class IncrementPolicy; - -template -class DatasetMapper; - -using DatasetInfo = DatasetMapper; - -} // namespace data -} // namespace mlpack - namespace mlpack { namespace util { diff --git a/src/mlpack/core/util/params.hpp b/src/mlpack/core/util/params.hpp index c6bb1596a7..b4791d6937 100644 --- a/src/mlpack/core/util/params.hpp +++ b/src/mlpack/core/util/params.hpp @@ -10,6 +10,7 @@ #include "forward.hpp" #include "param_data.hpp" #include "binding_details.hpp" +#include namespace mlpack { namespace util { From fe73ae2cf67fae1baedb49f7b33f0adc6481f777 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Jun 2022 18:33:29 -0400 Subject: [PATCH 24/39] Remove wayward definition. --- src/mlpack/base.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mlpack/base.hpp b/src/mlpack/base.hpp index 05aae68577..b71da0fd06 100644 --- a/src/mlpack/base.hpp +++ b/src/mlpack/base.hpp @@ -103,7 +103,6 @@ using enable_if_t = typename enable_if::type; // it's part of the C++11 standard. #ifdef _MSC_VER #pragma warning(disable : 4519) - #define ARMA_USE_CXX11 #endif // Ensure that the user isn't doing something stupid with their Armadillo From 0c5e0019878957df6d31cccf5562be788e2ccbd5 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Jun 2022 18:38:19 -0400 Subject: [PATCH 25/39] force_inline -> mlpack_force_inline --- src/mlpack/base.hpp | 10 +++++----- src/mlpack/methods/emst/dtb_rules_impl.hpp | 2 +- .../methods/fastmks/fastmks_rules_impl.hpp | 2 +- src/mlpack/methods/kde/kde_rules_impl.hpp | 18 +++++++++--------- .../methods/kmeans/allow_empty_clusters.hpp | 2 +- .../kmeans/dual_tree_kmeans_rules_impl.hpp | 3 ++- .../kmeans/pelleg_moore_kmeans_rules_impl.hpp | 2 +- src/mlpack/methods/lsh/lsh_search_impl.hpp | 12 ++++++------ .../neighbor_search_rules_impl.hpp | 2 +- .../range_search/range_search_rules_impl.hpp | 2 +- .../methods/rann/ra_search_rules_impl.hpp | 2 +- 11 files changed, 29 insertions(+), 28 deletions(-) diff --git a/src/mlpack/base.hpp b/src/mlpack/base.hpp index b71da0fd06..d4fef47884 100644 --- a/src/mlpack/base.hpp +++ b/src/mlpack/base.hpp @@ -56,13 +56,13 @@ #endif // Give ourselves a nice way to force functions to be inline if we need. -#define force_inline +#define mlpack_force_inline #if defined(__GNUG__) && !defined(DEBUG) - #undef force_inline - #define force_inline __attribute__((always_inline)) + #undef mlpack_force_inline + #define mlpack_force_inline __attribute__((always_inline)) #elif defined(_MSC_VER) && !defined(DEBUG) - #undef force_inline - #define force_inline __forceinline + #undef mlpack_force_inline + #define mlpack_force_inline __forceinline #endif // Backport this functionality from C++14, if it doesn't exist. diff --git a/src/mlpack/methods/emst/dtb_rules_impl.hpp b/src/mlpack/methods/emst/dtb_rules_impl.hpp index c7d7096d6b..88a6b59c97 100644 --- a/src/mlpack/methods/emst/dtb_rules_impl.hpp +++ b/src/mlpack/methods/emst/dtb_rules_impl.hpp @@ -37,7 +37,7 @@ DTBRules(const arma::mat& dataSet, } template -inline force_inline +inline mlpack_force_inline double DTBRules::BaseCase(const size_t queryIndex, const size_t referenceIndex) { diff --git a/src/mlpack/methods/fastmks/fastmks_rules_impl.hpp b/src/mlpack/methods/fastmks/fastmks_rules_impl.hpp index 898f8ba38b..567a348abc 100644 --- a/src/mlpack/methods/fastmks/fastmks_rules_impl.hpp +++ b/src/mlpack/methods/fastmks/fastmks_rules_impl.hpp @@ -82,7 +82,7 @@ void FastMKSRules::GetResults( } template -inline force_inline +inline mlpack_force_inline double FastMKSRules::BaseCase( const size_t queryIndex, const size_t referenceIndex) diff --git a/src/mlpack/methods/kde/kde_rules_impl.hpp b/src/mlpack/methods/kde/kde_rules_impl.hpp index 062e740efa..3c1faafa9e 100644 --- a/src/mlpack/methods/kde/kde_rules_impl.hpp +++ b/src/mlpack/methods/kde/kde_rules_impl.hpp @@ -66,7 +66,7 @@ KDERules::KDERules( //! The base case. template -inline force_inline +inline mlpack_force_inline double KDERules::BaseCase( const size_t queryIndex, const size_t referenceIndex) @@ -292,7 +292,7 @@ Score(const size_t queryIndex, TreeType& referenceNode) } template -inline force_inline double KDERules:: +inline mlpack_force_inline double KDERules:: Rescore(const size_t /* queryIndex */, TreeType& /* referenceNode */, const double oldScore) const @@ -515,7 +515,7 @@ Score(TreeType& queryNode, TreeType& referenceNode) //! Dual-tree rescore. template -inline force_inline double KDERules:: +inline mlpack_force_inline double KDERules:: Rescore(TreeType& /*queryNode*/, TreeType& /*referenceNode*/, const double oldScore) const @@ -525,7 +525,7 @@ Rescore(TreeType& /*queryNode*/, } template -inline force_inline double KDERules:: +inline mlpack_force_inline double KDERules:: EvaluateKernel(const size_t queryIndex, const size_t referenceIndex) const { @@ -534,14 +534,14 @@ EvaluateKernel(const size_t queryIndex, } template -inline force_inline double KDERules:: +inline mlpack_force_inline double KDERules:: EvaluateKernel(const arma::vec& query, const arma::vec& reference) const { return kernel.Evaluate(metric.Evaluate(query, reference)); } template -inline force_inline double KDERules:: +inline mlpack_force_inline double KDERules:: CalculateAlpha(TreeType* node) { KDEStat& stat = node->Stat(); @@ -571,7 +571,7 @@ CalculateAlpha(TreeType* node) //! Clean rules base case. template -inline force_inline +inline mlpack_force_inline double KDECleanRules::BaseCase(const size_t /* queryIndex */, const size_t /* refIndex */) { @@ -580,7 +580,7 @@ double KDECleanRules::BaseCase(const size_t /* queryIndex */, //! Clean rules single-tree score. template -inline force_inline +inline mlpack_force_inline double KDECleanRules::Score(const size_t /* queryIndex */, TreeType& referenceNode) { @@ -591,7 +591,7 @@ double KDECleanRules::Score(const size_t /* queryIndex */, //! Clean rules double-tree score. template -inline force_inline +inline mlpack_force_inline double KDECleanRules::Score(TreeType& queryNode, TreeType& referenceNode) { diff --git a/src/mlpack/methods/kmeans/allow_empty_clusters.hpp b/src/mlpack/methods/kmeans/allow_empty_clusters.hpp index ab0bdf34d0..3f65d06563 100644 --- a/src/mlpack/methods/kmeans/allow_empty_clusters.hpp +++ b/src/mlpack/methods/kmeans/allow_empty_clusters.hpp @@ -46,7 +46,7 @@ class AllowEmptyClusters * @return Number of points changed (0). */ template - static inline force_inline void EmptyCluster( + static inline mlpack_force_inline void EmptyCluster( const MatType& /* data */, const size_t emptyCluster, const arma::mat& oldCentroids, diff --git a/src/mlpack/methods/kmeans/dual_tree_kmeans_rules_impl.hpp b/src/mlpack/methods/kmeans/dual_tree_kmeans_rules_impl.hpp index 2d1a66fe12..ff60df7192 100644 --- a/src/mlpack/methods/kmeans/dual_tree_kmeans_rules_impl.hpp +++ b/src/mlpack/methods/kmeans/dual_tree_kmeans_rules_impl.hpp @@ -51,7 +51,8 @@ DualTreeKMeansRules::DualTreeKMeansRules( } template -inline force_inline double DualTreeKMeansRules::BaseCase( +inline mlpack_force_inline +double DualTreeKMeansRules::BaseCase( const size_t queryIndex, const size_t referenceIndex) { diff --git a/src/mlpack/methods/kmeans/pelleg_moore_kmeans_rules_impl.hpp b/src/mlpack/methods/kmeans/pelleg_moore_kmeans_rules_impl.hpp index 8e8f715636..617a71d763 100644 --- a/src/mlpack/methods/kmeans/pelleg_moore_kmeans_rules_impl.hpp +++ b/src/mlpack/methods/kmeans/pelleg_moore_kmeans_rules_impl.hpp @@ -38,7 +38,7 @@ PellegMooreKMeansRules::PellegMooreKMeansRules( } template -inline force_inline +inline mlpack_force_inline double PellegMooreKMeansRules::BaseCase( const size_t /* queryIndex */, const size_t /* referenceIndex */) diff --git a/src/mlpack/methods/lsh/lsh_search_impl.hpp b/src/mlpack/methods/lsh/lsh_search_impl.hpp index db0bb12bdc..ff9b33c351 100644 --- a/src/mlpack/methods/lsh/lsh_search_impl.hpp +++ b/src/mlpack/methods/lsh/lsh_search_impl.hpp @@ -352,7 +352,7 @@ void LSHSearch::Train(MatType referenceSet, // Base case where the query set is the reference set. (So, we can't return // ourselves as the nearest neighbor.) template -inline force_inline +inline mlpack_force_inline void LSHSearch::BaseCase( const size_t queryIndex, const arma::uvec& referenceIndices, @@ -398,7 +398,7 @@ void LSHSearch::BaseCase( // Base case for bichromatic search. template -inline force_inline +inline mlpack_force_inline void LSHSearch::BaseCase( const size_t queryIndex, const arma::uvec& referenceIndices, @@ -440,7 +440,7 @@ void LSHSearch::BaseCase( } template -inline force_inline +inline mlpack_force_inline double LSHSearch::PerturbationScore( const std::vector& A, const arma::vec& scores) const @@ -453,7 +453,7 @@ double LSHSearch::PerturbationScore( } template -inline force_inline +inline mlpack_force_inline bool LSHSearch::PerturbationShift( std::vector& A) const { @@ -472,7 +472,7 @@ bool LSHSearch::PerturbationShift( } template -inline force_inline +inline mlpack_force_inline bool LSHSearch::PerturbationExpand( std::vector& A) const { @@ -491,7 +491,7 @@ bool LSHSearch::PerturbationExpand( } template -inline force_inline +inline mlpack_force_inline bool LSHSearch::PerturbationValid( const std::vector& A) const { diff --git a/src/mlpack/methods/neighbor_search/neighbor_search_rules_impl.hpp b/src/mlpack/methods/neighbor_search/neighbor_search_rules_impl.hpp index 6bf8055efd..cd24cea8eb 100644 --- a/src/mlpack/methods/neighbor_search/neighbor_search_rules_impl.hpp +++ b/src/mlpack/methods/neighbor_search/neighbor_search_rules_impl.hpp @@ -80,7 +80,7 @@ void NeighborSearchRules::GetResults( }; template -inline force_inline // Absolutely MUST be inline so optimizations can happen. +inline mlpack_force_inline // Must be inline so optimizations can happen. double NeighborSearchRules:: BaseCase(const size_t queryIndex, const size_t referenceIndex) { diff --git a/src/mlpack/methods/range_search/range_search_rules_impl.hpp b/src/mlpack/methods/range_search/range_search_rules_impl.hpp index c9bbcfdbe2..098680f591 100644 --- a/src/mlpack/methods/range_search/range_search_rules_impl.hpp +++ b/src/mlpack/methods/range_search/range_search_rules_impl.hpp @@ -45,7 +45,7 @@ RangeSearchRules::RangeSearchRules( //! The base case. Evaluate the distance between the two points and add to the //! results if necessary. template -inline force_inline +inline mlpack_force_inline double RangeSearchRules::BaseCase( const size_t queryIndex, const size_t referenceIndex) diff --git a/src/mlpack/methods/rann/ra_search_rules_impl.hpp b/src/mlpack/methods/rann/ra_search_rules_impl.hpp index 8ed3ddb107..ff75d4be56 100644 --- a/src/mlpack/methods/rann/ra_search_rules_impl.hpp +++ b/src/mlpack/methods/rann/ra_search_rules_impl.hpp @@ -116,7 +116,7 @@ void RASearchRules::GetResults( }; template -inline force_inline +inline mlpack_force_inline double RASearchRules::BaseCase( const size_t queryIndex, const size_t referenceIndex) From 33fa90197c16e602cb6ca7ec347842bcda510ead Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Jun 2022 18:38:37 -0400 Subject: [PATCH 26/39] Oops, missed one. --- src/mlpack/methods/kmeans/kill_empty_clusters.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/kmeans/kill_empty_clusters.hpp b/src/mlpack/methods/kmeans/kill_empty_clusters.hpp index fa3ededd2e..dd20fd5cf8 100644 --- a/src/mlpack/methods/kmeans/kill_empty_clusters.hpp +++ b/src/mlpack/methods/kmeans/kill_empty_clusters.hpp @@ -46,7 +46,7 @@ class KillEmptyClusters * @return Number of points changed (0). */ template - static inline force_inline void EmptyCluster( + static inline mlpack_force_inline void EmptyCluster( const MatType& /* data */, const size_t emptyCluster, const arma::mat& /* oldCentroids */, From c02eba2ec9a551be7a5a2cd536c282f33777dde7 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Jun 2022 18:47:05 -0400 Subject: [PATCH 27/39] ANY -> MLPACK_ANY --- src/mlpack/base.hpp | 14 ++-- src/mlpack/bindings/R/R_option.hpp | 2 +- src/mlpack/bindings/R/default_param_impl.hpp | 6 +- src/mlpack/bindings/R/get_param.hpp | 2 +- src/mlpack/bindings/R/get_printable_param.hpp | 10 +-- src/mlpack/bindings/R/print_doc.hpp | 8 +-- src/mlpack/bindings/cli/add_to_cli11.hpp | 12 ++-- src/mlpack/bindings/cli/cli_option.hpp | 4 +- .../bindings/cli/default_param_impl.hpp | 6 +- .../bindings/cli/delete_allocated_memory.hpp | 2 +- .../bindings/cli/get_allocated_memory.hpp | 2 +- src/mlpack/bindings/cli/get_param.hpp | 8 +-- .../bindings/cli/get_printable_param_impl.hpp | 8 +-- src/mlpack/bindings/cli/get_raw_param.hpp | 6 +- src/mlpack/bindings/cli/in_place_copy.hpp | 8 +-- src/mlpack/bindings/cli/output_param_impl.hpp | 16 ++--- src/mlpack/bindings/cli/set_param.hpp | 18 ++--- src/mlpack/bindings/go/default_param_impl.hpp | 6 +- src/mlpack/bindings/go/get_param.hpp | 2 +- .../bindings/go/get_printable_param.hpp | 10 +-- src/mlpack/bindings/go/go_option.hpp | 2 +- src/mlpack/bindings/go/print_doc.hpp | 6 +- .../bindings/go/print_input_processing.hpp | 8 +-- src/mlpack/bindings/go/print_method_init.hpp | 8 +-- .../bindings/julia/default_param_impl.hpp | 6 +- src/mlpack/bindings/julia/get_param.hpp | 2 +- .../bindings/julia/get_printable_param.hpp | 10 +-- src/mlpack/bindings/julia/julia_option.hpp | 2 +- src/mlpack/bindings/julia/print_doc.hpp | 8 +-- src/mlpack/bindings/markdown/get_param.hpp | 2 +- .../bindings/markdown/get_printable_param.hpp | 10 +-- src/mlpack/bindings/markdown/md_option.hpp | 2 +- .../markdown/print_doc_functions_impl.hpp | 30 ++++---- .../bindings/python/default_param_impl.hpp | 6 +- src/mlpack/bindings/python/get_param.hpp | 2 +- .../bindings/python/get_printable_param.hpp | 10 +-- src/mlpack/bindings/python/py_option.hpp | 2 +- .../tests/delete_allocated_memory.hpp | 2 +- .../bindings/tests/get_allocated_memory.hpp | 2 +- src/mlpack/bindings/tests/get_param.hpp | 2 +- .../tests/get_printable_param_impl.hpp | 4 +- src/mlpack/bindings/tests/test_option.hpp | 2 +- src/mlpack/core/data/string_encoding.hpp | 8 +-- .../core/data/string_encoding_dictionary.hpp | 16 ++--- src/mlpack/core/data/string_encoding_impl.hpp | 8 +-- .../core/data/tokenizers/char_extract.hpp | 2 +- .../core/data/tokenizers/split_by_any_of.hpp | 16 ++--- src/mlpack/core/util/param_data.hpp | 2 +- src/mlpack/core/util/params_impl.hpp | 2 +- src/mlpack/tests/cli_binding_test.cpp | 68 +++++++++---------- src/mlpack/tests/python_binding_test.cpp | 4 +- src/mlpack/tests/string_encoding_test.cpp | 50 +++++++------- 52 files changed, 227 insertions(+), 227 deletions(-) diff --git a/src/mlpack/base.hpp b/src/mlpack/base.hpp index d4fef47884..e82946f9c9 100644 --- a/src/mlpack/base.hpp +++ b/src/mlpack/base.hpp @@ -83,16 +83,16 @@ using enable_if_t = typename enable_if::type; #if __cplusplus < 201703L #include #include - #define ANY core::v2::any - #define ANY_CAST core::v2::any_cast - #define STRING_VIEW core::v2::string_view + #define MLPACK_ANY core::v2::any + #define MLPACK_ANY_CAST core::v2::any_cast + #define MLPACK_STRING_VIEW core::v2::string_view #else #include #include - #define ANY std::any - #define ANY_CAST std::any_cast - #define STRING_VIEW std::string_view -#endif + #define MLPACK_ANY std::any + #define MLPACK_ANY_CAST std::any_cast + #define MLPACK_STRING_VIEW std::string_view +#endif // Now include Armadillo through the special mlpack extensions. #include diff --git a/src/mlpack/bindings/R/R_option.hpp b/src/mlpack/bindings/R/R_option.hpp index 5982494c50..a946052836 100644 --- a/src/mlpack/bindings/R/R_option.hpp +++ b/src/mlpack/bindings/R/R_option.hpp @@ -73,7 +73,7 @@ class ROption data.cppType = cppName; // Every parameter we'll get from R will have the correct type. - data.value = ANY(defaultValue); + data.value = MLPACK_ANY(defaultValue); // Set the function pointers that we'll need. All of these function // pointers will be used by both the program that generates the R, and diff --git a/src/mlpack/bindings/R/default_param_impl.hpp b/src/mlpack/bindings/R/default_param_impl.hpp index 64eea0d1e2..fd7e994200 100644 --- a/src/mlpack/bindings/R/default_param_impl.hpp +++ b/src/mlpack/bindings/R/default_param_impl.hpp @@ -36,7 +36,7 @@ std::string DefaultParamImpl( if (std::is_same::value) oss << "FALSE"; else - oss << ANY_CAST(data.value); + oss << MLPACK_ANY_CAST(data.value); return oss.str(); } @@ -51,7 +51,7 @@ std::string DefaultParamImpl( { // Print each element in an array delimited by square brackets. std::ostringstream oss; - const T& vector = ANY_CAST(data.value); + const T& vector = MLPACK_ANY_CAST(data.value); oss << "c("; if (std::is_same>::value) { @@ -92,7 +92,7 @@ std::string DefaultParamImpl( util::ParamData& data, const typename std::enable_if::value>::type*) { - const std::string& s = *ANY_CAST(&data.value); + const std::string& s = *MLPACK_ANY_CAST(&data.value); return "\"" + s + "\""; } diff --git a/src/mlpack/bindings/R/get_param.hpp b/src/mlpack/bindings/R/get_param.hpp index b725698c9e..72f6d3b311 100644 --- a/src/mlpack/bindings/R/get_param.hpp +++ b/src/mlpack/bindings/R/get_param.hpp @@ -27,7 +27,7 @@ void GetParam(util::ParamData& d, const void* /* input */, void* output) { - *((T**) output) = const_cast(ANY_CAST(&d.value)); + *((T**) output) = const_cast(MLPACK_ANY_CAST(&d.value)); } } // namespace r diff --git a/src/mlpack/bindings/R/get_printable_param.hpp b/src/mlpack/bindings/R/get_printable_param.hpp index 8f8fefaaaf..657dd18334 100644 --- a/src/mlpack/bindings/R/get_printable_param.hpp +++ b/src/mlpack/bindings/R/get_printable_param.hpp @@ -32,7 +32,7 @@ std::string GetPrintableParam( std::tuple>::value>::type* = 0) { std::ostringstream oss; - oss << ANY_CAST(data.value); + oss << MLPACK_ANY_CAST(data.value); return oss.str(); } @@ -44,7 +44,7 @@ std::string GetPrintableParam( util::ParamData& data, const typename std::enable_if::value>::type* = 0) { - const T& t = ANY_CAST(data.value); + const T& t = MLPACK_ANY_CAST(data.value); std::ostringstream oss; for (size_t i = 0; i < t.size(); ++i) @@ -61,7 +61,7 @@ std::string GetPrintableParam( const typename std::enable_if::value>::type* = 0) { // Get the matrix. - const T& matrix = ANY_CAST(data.value); + const T& matrix = MLPACK_ANY_CAST(data.value); std::ostringstream oss; oss << matrix.n_rows << "x" << matrix.n_cols << " matrix"; @@ -78,7 +78,7 @@ std::string GetPrintableParam( const typename std::enable_if::value>::type* = 0) { std::ostringstream oss; - oss << data.cppType << " model at " << ANY_CAST(data.value); + oss << data.cppType << " model at " << MLPACK_ANY_CAST(data.value); return oss.str(); } @@ -92,7 +92,7 @@ std::string GetPrintableParam( std::tuple>::value>::type* = 0) { // Get the matrix. - const T& tuple = ANY_CAST(data.value); + const T& tuple = MLPACK_ANY_CAST(data.value); const arma::mat& matrix = std::get<1>(tuple); std::ostringstream oss; diff --git a/src/mlpack/bindings/R/print_doc.hpp b/src/mlpack/bindings/R/print_doc.hpp index 2fa9a216bb..0c08a129a9 100644 --- a/src/mlpack/bindings/R/print_doc.hpp +++ b/src/mlpack/bindings/R/print_doc.hpp @@ -55,19 +55,19 @@ void PrintDoc(util::ParamData& d, oss << ". Default value \""; if (d.cppType == "std::string") { - oss << ANY_CAST(d.value); + oss << MLPACK_ANY_CAST(d.value); } else if (d.cppType == "double") { - oss << ANY_CAST(d.value); + oss << MLPACK_ANY_CAST(d.value); } else if (d.cppType == "int") { - oss << ANY_CAST(d.value); + oss << MLPACK_ANY_CAST(d.value); } else if (d.cppType == "bool") { - oss << (ANY_CAST(d.value) ? "TRUE" : "FALSE"); + oss << (MLPACK_ANY_CAST(d.value) ? "TRUE" : "FALSE"); } oss << "\""; } diff --git a/src/mlpack/bindings/cli/add_to_cli11.hpp b/src/mlpack/bindings/cli/add_to_cli11.hpp index 1e4e89f366..39de491b2e 100644 --- a/src/mlpack/bindings/cli/add_to_cli11.hpp +++ b/src/mlpack/bindings/cli/add_to_cli11.hpp @@ -47,8 +47,8 @@ void AddToCLI11(const std::string& cliName, [¶m](const std::string& value) { using TupleType = std::tuple::type>; - TupleType& tuple = *ANY_CAST(¶m.value); - std::get<0>(std::get<1>(tuple)) = ANY_CAST(value); + TupleType& tuple = *MLPACK_ANY_CAST(¶m.value); + std::get<0>(std::get<1>(tuple)) = MLPACK_ANY_CAST(value); param.wasPassed = true; }, param.desc.c_str()); @@ -79,8 +79,8 @@ void AddToCLI11(const std::string& cliName, [¶m](const std::string& value) { using TupleType = std::tuple::type>; - TupleType& tuple = *ANY_CAST(¶m.value); - std::get<1>(tuple) = ANY_CAST(value); + TupleType& tuple = *MLPACK_ANY_CAST(¶m.value); + std::get<1>(tuple) = MLPACK_ANY_CAST(value); param.wasPassed = true; }, param.desc.c_str()); @@ -109,8 +109,8 @@ void AddToCLI11(const std::string& cliName, [¶m](const std::string& value) { using TupleType = std::tuple::type>; - TupleType& tuple = *ANY_CAST(¶m.value); - std::get<0>(std::get<1>(tuple)) = ANY_CAST(value); + TupleType& tuple = *MLPACK_ANY_CAST(¶m.value); + std::get<0>(std::get<1>(tuple)) = MLPACK_ANY_CAST(value); param.wasPassed = true; }, param.desc.c_str()); diff --git a/src/mlpack/bindings/cli/cli_option.hpp b/src/mlpack/bindings/cli/cli_option.hpp index d8ea8e6182..ee041c0c22 100644 --- a/src/mlpack/bindings/cli/cli_option.hpp +++ b/src/mlpack/bindings/cli/cli_option.hpp @@ -95,12 +95,12 @@ class CLIOption typename ParameterType::type>::type>::value) { - data.value = ANY(defaultValue); + data.value = MLPACK_ANY(defaultValue); } else { typename ParameterType::type>::type tmp; - data.value = ANY(std::tuple(defaultValue, tmp)); + data.value = MLPACK_ANY(std::tuple(defaultValue, tmp)); } const std::string tname = data.tname; diff --git a/src/mlpack/bindings/cli/default_param_impl.hpp b/src/mlpack/bindings/cli/default_param_impl.hpp index ea0ec7cd7c..cae9a691c3 100644 --- a/src/mlpack/bindings/cli/default_param_impl.hpp +++ b/src/mlpack/bindings/cli/default_param_impl.hpp @@ -34,7 +34,7 @@ std::string DefaultParamImpl( { std::ostringstream oss; if (!std::is_same::value) - oss << ANY_CAST(data.value); + oss << MLPACK_ANY_CAST(data.value); return oss.str(); } @@ -49,7 +49,7 @@ std::string DefaultParamImpl( { // Print each element in an array delimited by square brackets. std::ostringstream oss; - const T& vector = ANY_CAST(data.value); + const T& vector = MLPACK_ANY_CAST(data.value); oss << "["; if (std::is_same>::value) { @@ -91,7 +91,7 @@ std::string DefaultParamImpl( util::ParamData& data, const typename std::enable_if::value>::type*) { - const std::string& s = *ANY_CAST(&data.value); + const std::string& s = *MLPACK_ANY_CAST(&data.value); return "'" + s + "'"; } diff --git a/src/mlpack/bindings/cli/delete_allocated_memory.hpp b/src/mlpack/bindings/cli/delete_allocated_memory.hpp index 2ba1681301..2aec6fc712 100644 --- a/src/mlpack/bindings/cli/delete_allocated_memory.hpp +++ b/src/mlpack/bindings/cli/delete_allocated_memory.hpp @@ -43,7 +43,7 @@ void DeleteAllocatedMemoryImpl( { // Delete the allocated memory (hopefully we actually own it). typedef std::tuple TupleType; - delete std::get<0>(*ANY_CAST(&d.value)); + delete std::get<0>(*MLPACK_ANY_CAST(&d.value)); } template diff --git a/src/mlpack/bindings/cli/get_allocated_memory.hpp b/src/mlpack/bindings/cli/get_allocated_memory.hpp index 82e4bf3b76..bb9fb931b4 100644 --- a/src/mlpack/bindings/cli/get_allocated_memory.hpp +++ b/src/mlpack/bindings/cli/get_allocated_memory.hpp @@ -45,7 +45,7 @@ void* GetAllocatedMemory( // Here we have a model, which is a tuple, and we need the address of the // memory. typedef std::tuple TupleType; - return std::get<0>(*ANY_CAST(&d.value)); + return std::get<0>(*MLPACK_ANY_CAST(&d.value)); } template diff --git a/src/mlpack/bindings/cli/get_param.hpp b/src/mlpack/bindings/cli/get_param.hpp index a2fd746187..d32d6655a2 100644 --- a/src/mlpack/bindings/cli/get_param.hpp +++ b/src/mlpack/bindings/cli/get_param.hpp @@ -34,7 +34,7 @@ T& GetParam( std::tuple>::value>::type* = 0) { // No mapping is needed, so just cast it directly. - return *ANY_CAST(&d.value); + return *MLPACK_ANY_CAST(&d.value); } /** @@ -52,7 +52,7 @@ T& GetParam( // times, but I am not bothered by that---it shouldn't be something that // happens. typedef std::tuple::type> TupleType; - TupleType& tuple = *ANY_CAST(&d.value); + TupleType& tuple = *MLPACK_ANY_CAST(&d.value); const std::string& value = std::get<0>(std::get<1>(tuple)); T& matrix = std::get<0>(tuple); size_t& n_rows = std::get<1>(std::get<1>(tuple)); @@ -86,7 +86,7 @@ T& GetParam( // If this is an input parameter, we need to load both the matrix and the // dataset info. typedef std::tuple> TupleType; - TupleType* tuple = ANY_CAST(&d.value); + TupleType* tuple = MLPACK_ANY_CAST(&d.value); const std::string& value = std::get<0>(std::get<1>(*tuple)); T& t = std::get<0>(*tuple); size_t& n_rows = std::get<1>(std::get<1>(*tuple)); @@ -116,7 +116,7 @@ T*& GetParam( // If the model is an input model, we have to load it from file. 'value' // contains the filename. typedef std::tuple TupleType; - TupleType* tuple = ANY_CAST(&d.value); + TupleType* tuple = MLPACK_ANY_CAST(&d.value); const std::string& value = std::get<1>(*tuple); if (d.input && !d.loaded) { diff --git a/src/mlpack/bindings/cli/get_printable_param_impl.hpp b/src/mlpack/bindings/cli/get_printable_param_impl.hpp index f6f303fce8..0c9875490d 100644 --- a/src/mlpack/bindings/cli/get_printable_param_impl.hpp +++ b/src/mlpack/bindings/cli/get_printable_param_impl.hpp @@ -30,7 +30,7 @@ std::string GetPrintableParam( std::tuple>::value>::type* /* junk */) { std::ostringstream oss; - oss << ANY_CAST(data.value); + oss << MLPACK_ANY_CAST(data.value); return oss.str(); } @@ -41,7 +41,7 @@ std::string GetPrintableParam( const typename std::enable_if::value>::type* /* junk */) { - const T& t = ANY_CAST(data.value); + const T& t = MLPACK_ANY_CAST(data.value); std::ostringstream oss; for (size_t i = 0; i < t.size(); ++i) @@ -80,7 +80,7 @@ std::string GetPrintableParam( { // Extract the string from the tuple that's being held. typedef std::tuple::type> TupleType; - const TupleType* tuple = ANY_CAST(&data.value); + const TupleType* tuple = MLPACK_ANY_CAST(&data.value); std::ostringstream oss; oss << "'" << std::get<0>(std::get<1>(*tuple)) << "'"; @@ -108,7 +108,7 @@ std::string GetPrintableParam( { // Extract the string from the tuple that's being held. typedef std::tuple::type> TupleType; - const TupleType* tuple = ANY_CAST(&data.value); + const TupleType* tuple = MLPACK_ANY_CAST(&data.value); std::ostringstream oss; oss << std::get<1>(*tuple); diff --git a/src/mlpack/bindings/cli/get_raw_param.hpp b/src/mlpack/bindings/cli/get_raw_param.hpp index b9aacd7f16..ea3c43f865 100644 --- a/src/mlpack/bindings/cli/get_raw_param.hpp +++ b/src/mlpack/bindings/cli/get_raw_param.hpp @@ -33,7 +33,7 @@ T& GetRawParam( std::tuple>::value>::type* = 0) { // No mapping is needed, so just cast it directly. - return *ANY_CAST(&d.value); + return *MLPACK_ANY_CAST(&d.value); } /** @@ -49,7 +49,7 @@ T& GetRawParam( { // Don't load the matrix. typedef std::tuple> TupleType; - T& value = std::get<0>(*ANY_CAST(&d.value)); + T& value = std::get<0>(*MLPACK_ANY_CAST(&d.value)); return value; } @@ -64,7 +64,7 @@ T*& GetRawParam( { // Don't load the model. typedef std::tuple TupleType; - T*& value = std::get<0>(*ANY_CAST(&d.value)); + T*& value = std::get<0>(*MLPACK_ANY_CAST(&d.value)); return value; } diff --git a/src/mlpack/bindings/cli/in_place_copy.hpp b/src/mlpack/bindings/cli/in_place_copy.hpp index a787045eca..742058b1c5 100644 --- a/src/mlpack/bindings/cli/in_place_copy.hpp +++ b/src/mlpack/bindings/cli/in_place_copy.hpp @@ -58,10 +58,10 @@ void InPlaceCopyInternal( { // Make the output filename the same as the input filename. typedef std::tuple::type> TupleType; - TupleType& tuple = *ANY_CAST(&d.value); + TupleType& tuple = *MLPACK_ANY_CAST(&d.value); std::string& value = std::get<0>(std::get<1>(tuple)); - const TupleType& inputTuple = *ANY_CAST(&input.value); + const TupleType& inputTuple = *MLPACK_ANY_CAST(&input.value); value = std::get<0>(std::get<1>(inputTuple)); } @@ -81,10 +81,10 @@ void InPlaceCopyInternal( { // Make the output filename the same as the input filename. typedef std::tuple::type> TupleType; - TupleType& tuple = *ANY_CAST(&d.value); + TupleType& tuple = *MLPACK_ANY_CAST(&d.value); std::string& value = std::get<1>(tuple); - const TupleType& inputTuple = *ANY_CAST(&input.value); + const TupleType& inputTuple = *MLPACK_ANY_CAST(&input.value); value = std::get<1>(inputTuple); } diff --git a/src/mlpack/bindings/cli/output_param_impl.hpp b/src/mlpack/bindings/cli/output_param_impl.hpp index e1cd1f411a..e9892643b1 100644 --- a/src/mlpack/bindings/cli/output_param_impl.hpp +++ b/src/mlpack/bindings/cli/output_param_impl.hpp @@ -30,7 +30,7 @@ void OutputParamImpl( const typename std::enable_if>::value>::type* /* junk */) { - std::cout << data.name << ": " << *ANY_CAST(&data.value) + std::cout << data.name << ": " << *MLPACK_ANY_CAST(&data.value) << std::endl; } @@ -41,7 +41,7 @@ void OutputParamImpl( const typename std::enable_if::value>::type* /* junk */) { std::cout << data.name << ": "; - const T& t = *ANY_CAST(&data.value); + const T& t = *MLPACK_ANY_CAST(&data.value); for (size_t i = 0; i < t.size(); ++i) std::cout << t[i] << " "; std::cout << std::endl; @@ -54,9 +54,9 @@ void OutputParamImpl( const typename std::enable_if::value>::type* /* junk */) { typedef std::tuple> TupleType; - const T& output = std::get<0>(*ANY_CAST(&data.value)); + const T& output = std::get<0>(*MLPACK_ANY_CAST(&data.value)); const std::string& filename = - std::get<0>(std::get<1>(*ANY_CAST(&data.value))); + std::get<0>(std::get<1>(*MLPACK_ANY_CAST(&data.value))); if (output.n_elem > 0 && filename != "") { @@ -78,10 +78,10 @@ void OutputParamImpl( // const. In this case we can assume it though, since we will be saving and // not loading. typedef std::tuple TupleType; - T*& output = const_cast(std::get<0>(*ANY_CAST( + T*& output = const_cast(std::get<0>(*MLPACK_ANY_CAST( &data.value))); const std::string& filename = - std::get<1>(*ANY_CAST(&data.value)); + std::get<1>(*MLPACK_ANY_CAST(&data.value)); if (filename != "") data::Save(filename, "model", *output); @@ -96,9 +96,9 @@ void OutputParamImpl( { // Output the matrix with the mappings. typedef std::tuple> TupleType; - const T& tuple = std::get<0>(*ANY_CAST(&data.value)); + const T& tuple = std::get<0>(*MLPACK_ANY_CAST(&data.value)); const std::string& filename = - std::get<0>(std::get<1>(*ANY_CAST(&data.value))); + std::get<0>(std::get<1>(*MLPACK_ANY_CAST(&data.value))); const arma::mat& matrix = std::get<1>(tuple); // The mapping isn't taken into account. We should write a data::Save() diff --git a/src/mlpack/bindings/cli/set_param.hpp b/src/mlpack/bindings/cli/set_param.hpp index cab459dd7c..d575896c8d 100644 --- a/src/mlpack/bindings/cli/set_param.hpp +++ b/src/mlpack/bindings/cli/set_param.hpp @@ -26,7 +26,7 @@ namespace cli { template void SetParam( util::ParamData& d, - const ANY& value, + const MLPACK_ANY& value, const typename std::enable_if::value>::type* = 0, const typename std::enable_if::value>::type* = 0, const typename std::enable_if void SetParam( util::ParamData& d, - const ANY& /* value */, + const MLPACK_ANY& /* value */, const typename std::enable_if::value>::type* = 0) { // Force set to the value of whether or not this was passed. @@ -57,15 +57,15 @@ void SetParam( template void SetParam( util::ParamData& d, - const ANY& value, + const MLPACK_ANY& value, const typename std::enable_if::value || std::is_same>::value>::type* = 0) { // We're setting the string filename. typedef std::tuple::type> TupleType; - TupleType& tuple = *ANY_CAST(&d.value); - std::get<0>(std::get<1>(tuple)) = ANY_CAST(value); + TupleType& tuple = *MLPACK_ANY_CAST(&d.value); + std::get<0>(std::get<1>(tuple)) = MLPACK_ANY_CAST(value); } /** @@ -75,14 +75,14 @@ void SetParam( template void SetParam( util::ParamData& d, - const ANY& value, + const MLPACK_ANY& value, const typename std::enable_if::value>::type* = 0, const typename std::enable_if::value>::type* = 0) { // We're setting the string filename. typedef std::tuple::type> TupleType; - TupleType& tuple = *ANY_CAST(&d.value); - std::get<1>(tuple) = ANY_CAST(value); + TupleType& tuple = *MLPACK_ANY_CAST(&d.value); + std::get<1>(tuple) = MLPACK_ANY_CAST(value); } /** @@ -97,7 +97,7 @@ template void SetParam(util::ParamData& d, const void* input, void* /* output */) { SetParam::type>( - const_cast(d), *((ANY*) input)); + const_cast(d), *((MLPACK_ANY*) input)); } } // namespace cli diff --git a/src/mlpack/bindings/go/default_param_impl.hpp b/src/mlpack/bindings/go/default_param_impl.hpp index 0bae6ce306..1be5d696be 100644 --- a/src/mlpack/bindings/go/default_param_impl.hpp +++ b/src/mlpack/bindings/go/default_param_impl.hpp @@ -36,7 +36,7 @@ std::string DefaultParamImpl( if (std::is_same::value) oss << "false"; else - oss << ANY_CAST(data.value); + oss << MLPACK_ANY_CAST(data.value); return oss.str(); } @@ -51,7 +51,7 @@ std::string DefaultParamImpl( { // Print each element in an array delimited by square brackets. std::ostringstream oss; - const T& vector = ANY_CAST(data.value); + const T& vector = MLPACK_ANY_CAST(data.value); if (std::is_same>::value) { oss << "[]string{"; @@ -93,7 +93,7 @@ std::string DefaultParamImpl( util::ParamData& data, const typename std::enable_if::value>::type*) { - const std::string& s = *ANY_CAST(&data.value); + const std::string& s = *MLPACK_ANY_CAST(&data.value); return "\"" + s + "\""; } diff --git a/src/mlpack/bindings/go/get_param.hpp b/src/mlpack/bindings/go/get_param.hpp index 8ef97d31e1..9c02ab8e77 100644 --- a/src/mlpack/bindings/go/get_param.hpp +++ b/src/mlpack/bindings/go/get_param.hpp @@ -27,7 +27,7 @@ void GetParam(util::ParamData& d, const void* /* input */, void* output) { - *((T**) output) = const_cast(ANY_CAST(&d.value)); + *((T**) output) = const_cast(MLPACK_ANY_CAST(&d.value)); } } // namespace go diff --git a/src/mlpack/bindings/go/get_printable_param.hpp b/src/mlpack/bindings/go/get_printable_param.hpp index fb46da993d..76d3301192 100644 --- a/src/mlpack/bindings/go/get_printable_param.hpp +++ b/src/mlpack/bindings/go/get_printable_param.hpp @@ -32,7 +32,7 @@ std::string GetPrintableParam( std::tuple>::value>::type* = 0) { std::ostringstream oss; - oss << ANY_CAST(data.value); + oss << MLPACK_ANY_CAST(data.value); return oss.str(); } @@ -44,7 +44,7 @@ std::string GetPrintableParam( util::ParamData& data, const typename std::enable_if::value>::type* = 0) { - const T& t = ANY_CAST(data.value); + const T& t = MLPACK_ANY_CAST(data.value); std::ostringstream oss; for (size_t i = 0; i < t.size(); ++i) @@ -61,7 +61,7 @@ std::string GetPrintableParam( const typename std::enable_if::value>::type* = 0) { // Get the matrix. - const T& matrix = ANY_CAST(data.value); + const T& matrix = MLPACK_ANY_CAST(data.value); std::ostringstream oss; oss << matrix.n_rows << "x" << matrix.n_cols << " matrix"; @@ -78,7 +78,7 @@ std::string GetPrintableParam( const typename std::enable_if::value>::type* = 0) { std::ostringstream oss; - oss << data.cppType << " model at " << ANY_CAST(data.value); + oss << data.cppType << " model at " << MLPACK_ANY_CAST(data.value); return oss.str(); } @@ -92,7 +92,7 @@ std::string GetPrintableParam( std::tuple>::value>::type* = 0) { // Get the matrix. - const T& tuple = ANY_CAST(data.value); + const T& tuple = MLPACK_ANY_CAST(data.value); const arma::mat& matrix = std::get<1>(tuple); std::ostringstream oss; diff --git a/src/mlpack/bindings/go/go_option.hpp b/src/mlpack/bindings/go/go_option.hpp index 2554bc403e..5290e2738c 100644 --- a/src/mlpack/bindings/go/go_option.hpp +++ b/src/mlpack/bindings/go/go_option.hpp @@ -78,7 +78,7 @@ class GoOption data.loaded = false; data.cppType = cppName; - data.value = ANY(defaultValue); + data.value = MLPACK_ANY(defaultValue); // Set the function pointers that we'll need. All of these function // pointers will be used by both the program that generates the .cpp, diff --git a/src/mlpack/bindings/go/print_doc.hpp b/src/mlpack/bindings/go/print_doc.hpp index d312f602f1..c12aa83a62 100644 --- a/src/mlpack/bindings/go/print_doc.hpp +++ b/src/mlpack/bindings/go/print_doc.hpp @@ -53,16 +53,16 @@ void PrintDoc(util::ParamData& d, { if (d.cppType == "std::string") { - oss << " Default value '" << ANY_CAST(d.value) + oss << " Default value '" << MLPACK_ANY_CAST(d.value) << "'."; } else if (d.cppType == "double") { - oss << " Default value " << ANY_CAST(d.value) << "."; + oss << " Default value " << MLPACK_ANY_CAST(d.value) << "."; } else if (d.cppType == "int") { - oss << " Default value " << ANY_CAST(d.value) << "."; + oss << " Default value " << MLPACK_ANY_CAST(d.value) << "."; } } diff --git a/src/mlpack/bindings/go/print_input_processing.hpp b/src/mlpack/bindings/go/print_input_processing.hpp index 880770c3b6..a337d8b577 100644 --- a/src/mlpack/bindings/go/print_input_processing.hpp +++ b/src/mlpack/bindings/go/print_input_processing.hpp @@ -67,22 +67,22 @@ void PrintInputProcessing( // Print out default value. if (d.cppType == "std::string") { - std::string value = ANY_CAST(d.value); + std::string value = MLPACK_ANY_CAST(d.value); std::cout << "\"" << value << "\""; } else if (d.cppType == "double") { - double value = ANY_CAST(d.value); + double value = MLPACK_ANY_CAST(d.value); std::cout << value; } else if (d.cppType == "int") { - int value = ANY_CAST(d.value); + int value = MLPACK_ANY_CAST(d.value); std::cout << value; } else if (d.cppType == "bool") { - bool value = ANY_CAST(d.value); + bool value = MLPACK_ANY_CAST(d.value); if (value == 0) std::cout << "false"; else diff --git a/src/mlpack/bindings/go/print_method_init.hpp b/src/mlpack/bindings/go/print_method_init.hpp index 0e2c055d5b..7c42bfdcab 100644 --- a/src/mlpack/bindings/go/print_method_init.hpp +++ b/src/mlpack/bindings/go/print_method_init.hpp @@ -54,23 +54,23 @@ void PrintMethodInit( { if (d.cppType == "std::string") { - std::string value = ANY_CAST(d.value); + std::string value = MLPACK_ANY_CAST(d.value); std::cout << prefix << goParamName << ": \"" << value << "\"," << std::endl; } else if (d.cppType == "double") { - double value = ANY_CAST(d.value); + double value = MLPACK_ANY_CAST(d.value); std::cout << prefix << goParamName << ": " << value << "," << std::endl; } else if (d.cppType == "int") { - int value = ANY_CAST(d.value); + int value = MLPACK_ANY_CAST(d.value); std::cout << prefix << goParamName << ": " << value << "," << std::endl; } else if (d.cppType == "bool") { - bool value = ANY_CAST(d.value); + bool value = MLPACK_ANY_CAST(d.value); if (value == 0) std::cout << prefix << goParamName << ": false," << std::endl; else diff --git a/src/mlpack/bindings/julia/default_param_impl.hpp b/src/mlpack/bindings/julia/default_param_impl.hpp index d36b146608..1a2b8ca954 100644 --- a/src/mlpack/bindings/julia/default_param_impl.hpp +++ b/src/mlpack/bindings/julia/default_param_impl.hpp @@ -36,7 +36,7 @@ std::string DefaultParamImpl( if (std::is_same::value) oss << "false"; else - oss << ANY_CAST(data.value); + oss << MLPACK_ANY_CAST(data.value); return oss.str(); } @@ -51,7 +51,7 @@ std::string DefaultParamImpl( { // Print each element in an array delimited by square brackets. std::ostringstream oss; - const T& vector = ANY_CAST(data.value); + const T& vector = MLPACK_ANY_CAST(data.value); oss << "["; if (std::is_same>::value) { @@ -92,7 +92,7 @@ std::string DefaultParamImpl( util::ParamData& data, const typename std::enable_if::value>::type*) { - const std::string& s = *ANY_CAST(&data.value); + const std::string& s = *MLPACK_ANY_CAST(&data.value); return "\"" + s + "\""; } diff --git a/src/mlpack/bindings/julia/get_param.hpp b/src/mlpack/bindings/julia/get_param.hpp index 53983b88b8..4d493f7113 100644 --- a/src/mlpack/bindings/julia/get_param.hpp +++ b/src/mlpack/bindings/julia/get_param.hpp @@ -27,7 +27,7 @@ void GetParam(util::ParamData& d, const void* /* input */, void* output) { - *((T**) output) = const_cast(ANY_CAST(&d.value)); + *((T**) output) = const_cast(MLPACK_ANY_CAST(&d.value)); } } // namespace julia diff --git a/src/mlpack/bindings/julia/get_printable_param.hpp b/src/mlpack/bindings/julia/get_printable_param.hpp index 0f204f02ea..4fbb0e6859 100644 --- a/src/mlpack/bindings/julia/get_printable_param.hpp +++ b/src/mlpack/bindings/julia/get_printable_param.hpp @@ -32,7 +32,7 @@ std::string GetPrintableParam( std::tuple>::value>::type* = 0) { std::ostringstream oss; - oss << ANY_CAST(data.value); + oss << MLPACK_ANY_CAST(data.value); return oss.str(); } @@ -44,7 +44,7 @@ std::string GetPrintableParam( util::ParamData& data, const typename std::enable_if::value>::type* = 0) { - const T& t = ANY_CAST(data.value); + const T& t = MLPACK_ANY_CAST(data.value); std::ostringstream oss; for (size_t i = 0; i < t.size(); ++i) @@ -61,7 +61,7 @@ std::string GetPrintableParam( const typename std::enable_if::value>::type* = 0) { // Get the matrix. - const T& matrix = ANY_CAST(data.value); + const T& matrix = MLPACK_ANY_CAST(data.value); std::ostringstream oss; oss << matrix.n_rows << "x" << matrix.n_cols << " matrix"; @@ -78,7 +78,7 @@ std::string GetPrintableParam( const typename std::enable_if::value>::type* = 0) { std::ostringstream oss; - oss << data.cppType << " model at " << ANY_CAST(data.value); + oss << data.cppType << " model at " << MLPACK_ANY_CAST(data.value); return oss.str(); } @@ -92,7 +92,7 @@ std::string GetPrintableParam( std::tuple>::value>::type* = 0) { // Get the matrix. - const T& tuple = ANY_CAST(data.value); + const T& tuple = MLPACK_ANY_CAST(data.value); const arma::mat& matrix = std::get<1>(tuple); std::ostringstream oss; diff --git a/src/mlpack/bindings/julia/julia_option.hpp b/src/mlpack/bindings/julia/julia_option.hpp index 1a1bcea4b2..d8adeb13e3 100644 --- a/src/mlpack/bindings/julia/julia_option.hpp +++ b/src/mlpack/bindings/julia/julia_option.hpp @@ -64,7 +64,7 @@ class JuliaOption data.cppType = cppName; // Every parameter we'll get from Julia will have the correct type. - data.value = ANY(defaultValue); + data.value = MLPACK_ANY(defaultValue); // Set the function pointers that we'll need. All of these function // pointers will be used by both the program that generates the pyx, and diff --git a/src/mlpack/bindings/julia/print_doc.hpp b/src/mlpack/bindings/julia/print_doc.hpp index 30d887137c..c63a9d6872 100644 --- a/src/mlpack/bindings/julia/print_doc.hpp +++ b/src/mlpack/bindings/julia/print_doc.hpp @@ -39,19 +39,19 @@ void PrintDoc(util::ParamData& d, const void* /* input */, void* output) oss << " Default value `"; if (d.cppType == "std::string") { - oss << ANY_CAST(d.value); + oss << MLPACK_ANY_CAST(d.value); } else if (d.cppType == "double") { - oss << ANY_CAST(d.value); + oss << MLPACK_ANY_CAST(d.value); } else if (d.cppType == "int") { - oss << ANY_CAST(d.value); + oss << MLPACK_ANY_CAST(d.value); } else if (d.cppType == "bool") { - oss << (ANY_CAST(d.value) ? "true" : "false"); + oss << (MLPACK_ANY_CAST(d.value) ? "true" : "false"); } oss << "`." << std::endl; } diff --git a/src/mlpack/bindings/markdown/get_param.hpp b/src/mlpack/bindings/markdown/get_param.hpp index 748798355c..40176677c0 100644 --- a/src/mlpack/bindings/markdown/get_param.hpp +++ b/src/mlpack/bindings/markdown/get_param.hpp @@ -28,7 +28,7 @@ void GetParam(util::ParamData& d, void* output) { util::ParamData& dmod = const_cast(d); - *((T**) output) = ANY_CAST(&dmod.value); + *((T**) output) = MLPACK_ANY_CAST(&dmod.value); } } // namespace markdown diff --git a/src/mlpack/bindings/markdown/get_printable_param.hpp b/src/mlpack/bindings/markdown/get_printable_param.hpp index 068d4c8ce9..39e2c435d3 100644 --- a/src/mlpack/bindings/markdown/get_printable_param.hpp +++ b/src/mlpack/bindings/markdown/get_printable_param.hpp @@ -32,7 +32,7 @@ std::string GetPrintableParam( std::tuple>::value>::type* = 0) { std::ostringstream oss; - oss << ANY_CAST(data.value); + oss << MLPACK_ANY_CAST(data.value); return oss.str(); } @@ -44,7 +44,7 @@ std::string GetPrintableParam( util::ParamData& data, const typename std::enable_if::value>::type* = 0) { - const T& t = ANY_CAST(data.value); + const T& t = MLPACK_ANY_CAST(data.value); std::ostringstream oss; for (size_t i = 0; i < t.size(); ++i) @@ -61,7 +61,7 @@ std::string GetPrintableParam( const typename std::enable_if::value>::type* = 0) { // Get the matrix. - const T& matrix = ANY_CAST(data.value); + const T& matrix = MLPACK_ANY_CAST(data.value); std::ostringstream oss; oss << matrix.n_rows << "x" << matrix.n_cols << " matrix"; @@ -78,7 +78,7 @@ std::string GetPrintableParam( const typename std::enable_if::value>::type* = 0) { std::ostringstream oss; - oss << data.cppType << " model at " << ANY_CAST(data.value); + oss << data.cppType << " model at " << MLPACK_ANY_CAST(data.value); return oss.str(); } @@ -92,7 +92,7 @@ std::string GetPrintableParam( std::tuple>::value>::type* = 0) { // Get the matrix. - const T& tuple = ANY_CAST(data.value); + const T& tuple = MLPACK_ANY_CAST(data.value); const arma::mat& matrix = std::get<1>(tuple); std::ostringstream oss; diff --git a/src/mlpack/bindings/markdown/md_option.hpp b/src/mlpack/bindings/markdown/md_option.hpp index e8d4db8da2..aecbdb785e 100644 --- a/src/mlpack/bindings/markdown/md_option.hpp +++ b/src/mlpack/bindings/markdown/md_option.hpp @@ -63,7 +63,7 @@ class MDOption data.cppType = cppName; // Every parameter we'll get from Markdown will have the correct type. - data.value = ANY(defaultValue); + data.value = MLPACK_ANY(defaultValue); // Set the function pointers that we'll need. Most of these simply delegate // to the current binding type's implementation. Any new language will need diff --git a/src/mlpack/bindings/markdown/print_doc_functions_impl.hpp b/src/mlpack/bindings/markdown/print_doc_functions_impl.hpp index e298d6088e..38c5037a06 100644 --- a/src/mlpack/bindings/markdown/print_doc_functions_impl.hpp +++ b/src/mlpack/bindings/markdown/print_doc_functions_impl.hpp @@ -263,7 +263,7 @@ inline std::string PrintTypeDocs() data.required = false; data.input = true; data.loaded = false; - data.value = ANY(int(0)); + data.value = MLPACK_ANY(int(0)); std::string type = GetPrintableType(data); oss << " - `" << type << "`{: #doc_" << BindingInfo::Language() << "_" @@ -271,7 +271,7 @@ inline std::string PrintTypeDocs() data.tname = std::string(typeid(double).name()); data.cppType = "double"; - data.value = ANY(double(0.0)); + data.value = MLPACK_ANY(double(0.0)); type = GetPrintableType(data); oss << " - `" << type << "`{: #doc_" << BindingInfo::Language() << "_" @@ -280,7 +280,7 @@ inline std::string PrintTypeDocs() data.tname = std::string(typeid(bool).name()); data.cppType = "double"; - data.value = ANY(bool(0.0)); + data.value = MLPACK_ANY(bool(0.0)); type = GetPrintableType(data); oss << " - `" << type << "`{: #doc_" << BindingInfo::Language() << "_" @@ -288,7 +288,7 @@ inline std::string PrintTypeDocs() data.tname = std::string(typeid(std::string).name()); data.cppType = "std::string"; - data.value = ANY(std::string("")); + data.value = MLPACK_ANY(std::string("")); type = GetPrintableType(data); oss << " - `" << type << "`{: #doc_" << BindingInfo::Language() << "_" @@ -297,7 +297,7 @@ inline std::string PrintTypeDocs() data.tname = std::string(typeid(std::vector).name()); data.cppType = "std::vector"; - data.value = ANY(std::vector()); + data.value = MLPACK_ANY(std::vector()); type = GetPrintableType>(data); oss << " - `" << type << "`{: #doc_" << BindingInfo::Language() << "_" @@ -306,7 +306,7 @@ inline std::string PrintTypeDocs() data.tname = std::string(typeid(std::vector).name()); data.cppType = "std::vector"; - data.value = ANY(std::vector()); + data.value = MLPACK_ANY(std::vector()); type = GetPrintableType>(data); oss << " - `" << type << "`{: " << "#doc_" << BindingInfo::Language() << "_" @@ -315,7 +315,7 @@ inline std::string PrintTypeDocs() data.tname = std::string(typeid(arma::mat).name()); data.cppType = "arma::mat"; - data.value = ANY(arma::mat()); + data.value = MLPACK_ANY(arma::mat()); type = GetPrintableType(data); oss << " - `" << type << "`{: #doc_" << BindingInfo::Language() << "_" @@ -324,7 +324,7 @@ inline std::string PrintTypeDocs() data.tname = std::string(typeid(arma::Mat).name()); data.cppType = "arma::Mat"; - data.value = ANY(arma::Mat()); + data.value = MLPACK_ANY(arma::Mat()); type = GetPrintableType>(data); oss << " - `" << type << "`{: #doc_" << BindingInfo::Language() << "_" @@ -333,7 +333,7 @@ inline std::string PrintTypeDocs() data.tname = std::string(typeid(arma::rowvec).name()); data.cppType = "arma::rowvec"; - data.value = ANY(arma::rowvec()); + data.value = MLPACK_ANY(arma::rowvec()); const std::string& rowType = GetPrintableType(data); oss << " - `" << rowType << "`{: #doc_" << BindingInfo::Language() << "_" @@ -342,7 +342,7 @@ inline std::string PrintTypeDocs() data.tname = std::string(typeid(arma::Row).name()); data.cppType = "arma::Row"; - data.value = ANY(arma::Row()); + data.value = MLPACK_ANY(arma::Row()); const std::string& urowType = GetPrintableType>(data); oss << " - `" << urowType << "`{: #doc_" << BindingInfo::Language() << "_" @@ -352,7 +352,7 @@ inline std::string PrintTypeDocs() data.tname = std::string(typeid(arma::vec).name()); data.cppType = "arma::vec"; - data.value = ANY(arma::vec()); + data.value = MLPACK_ANY(arma::vec()); const std::string& colType = GetPrintableType(data); // For some languages there is no distinction between column and row vectors. @@ -366,7 +366,7 @@ inline std::string PrintTypeDocs() data.tname = std::string(typeid(arma::Col).name()); data.cppType = "arma::Col"; - data.value = ANY(arma::Col()); + data.value = MLPACK_ANY(arma::Col()); const std::string& ucolType = GetPrintableType>(data); // For some languages there is no distinction between column and row vectors. @@ -381,7 +381,7 @@ inline std::string PrintTypeDocs() data.tname = std::string(typeid(std::tuple).name()); data.cppType = "std::tuple"; - data.value = ANY(std::tuple()); + data.value = MLPACK_ANY(std::tuple()); type = GetPrintableType>(data); oss << " - `" << type << "`{: #doc_" << BindingInfo::Language() << "_" @@ -391,14 +391,14 @@ inline std::string PrintTypeDocs() data.tname = std::string(typeid(priv::mlpackModel).name()); data.cppType = "mlpackModel"; - data.value = ANY(new priv::mlpackModel()); + data.value = MLPACK_ANY(new priv::mlpackModel()); type = GetPrintableType(data); oss << " - `" << type << "`{: #doc_" << BindingInfo::Language() << "_model }: " << PrintTypeDoc(data) << std::endl; // Clean up memory. - delete ANY_CAST(data.value); + delete MLPACK_ANY_CAST(data.value); oss << std::endl << "" << std::endl; diff --git a/src/mlpack/bindings/python/default_param_impl.hpp b/src/mlpack/bindings/python/default_param_impl.hpp index b370a97fda..d032f4fa52 100644 --- a/src/mlpack/bindings/python/default_param_impl.hpp +++ b/src/mlpack/bindings/python/default_param_impl.hpp @@ -36,7 +36,7 @@ std::string DefaultParamImpl( if (std::is_same::value) oss << "False"; else - oss << ANY_CAST(data.value); + oss << MLPACK_ANY_CAST(data.value); return oss.str(); } @@ -51,7 +51,7 @@ std::string DefaultParamImpl( { // Print each element in an array delimited by square brackets. std::ostringstream oss; - const T& vector = ANY_CAST(data.value); + const T& vector = MLPACK_ANY_CAST(data.value); oss << "["; if (std::is_same>::value) { @@ -92,7 +92,7 @@ std::string DefaultParamImpl( util::ParamData& data, const typename std::enable_if::value>::type*) { - const std::string& s = *ANY_CAST(&data.value); + const std::string& s = *MLPACK_ANY_CAST(&data.value); return "'" + s + "'"; } diff --git a/src/mlpack/bindings/python/get_param.hpp b/src/mlpack/bindings/python/get_param.hpp index 8f9657ea53..7cbfb947c4 100644 --- a/src/mlpack/bindings/python/get_param.hpp +++ b/src/mlpack/bindings/python/get_param.hpp @@ -27,7 +27,7 @@ void GetParam(util::ParamData& d, const void* /* input */, void* output) { - *((T**) output) = const_cast(ANY_CAST(&d.value)); + *((T**) output) = const_cast(MLPACK_ANY_CAST(&d.value)); } } // namespace python diff --git a/src/mlpack/bindings/python/get_printable_param.hpp b/src/mlpack/bindings/python/get_printable_param.hpp index b9c7993365..1a67510f2a 100644 --- a/src/mlpack/bindings/python/get_printable_param.hpp +++ b/src/mlpack/bindings/python/get_printable_param.hpp @@ -32,7 +32,7 @@ std::string GetPrintableParam( std::tuple>::value>::type* = 0) { std::ostringstream oss; - oss << ANY_CAST(data.value); + oss << MLPACK_ANY_CAST(data.value); return oss.str(); } @@ -44,7 +44,7 @@ std::string GetPrintableParam( util::ParamData& data, const typename std::enable_if::value>::type* = 0) { - const T& t = ANY_CAST(data.value); + const T& t = MLPACK_ANY_CAST(data.value); std::ostringstream oss; for (size_t i = 0; i < t.size(); ++i) @@ -61,7 +61,7 @@ std::string GetPrintableParam( const typename std::enable_if::value>::type* = 0) { // Get the matrix. - const T& matrix = ANY_CAST(data.value); + const T& matrix = MLPACK_ANY_CAST(data.value); std::ostringstream oss; oss << matrix.n_rows << "x" << matrix.n_cols << " matrix"; @@ -78,7 +78,7 @@ std::string GetPrintableParam( const typename std::enable_if::value>::type* = 0) { std::ostringstream oss; - oss << data.cppType << " model at " << ANY_CAST(data.value); + oss << data.cppType << " model at " << MLPACK_ANY_CAST(data.value); return oss.str(); } @@ -92,7 +92,7 @@ std::string GetPrintableParam( std::tuple>::value>::type* = 0) { // Get the matrix. - const T& tuple = ANY_CAST(data.value); + const T& tuple = MLPACK_ANY_CAST(data.value); const arma::mat& matrix = std::get<1>(tuple); std::ostringstream oss; diff --git a/src/mlpack/bindings/python/py_option.hpp b/src/mlpack/bindings/python/py_option.hpp index d529d85ea5..5c92eeee7c 100644 --- a/src/mlpack/bindings/python/py_option.hpp +++ b/src/mlpack/bindings/python/py_option.hpp @@ -64,7 +64,7 @@ class PyOption data.cppType = cppName; // Every parameter we'll get from Python will have the correct type. - data.value = ANY(defaultValue); + data.value = MLPACK_ANY(defaultValue); // Set the function pointers that we'll need. All of these function // pointers will be used by both the program that generates the pyx, and diff --git a/src/mlpack/bindings/tests/delete_allocated_memory.hpp b/src/mlpack/bindings/tests/delete_allocated_memory.hpp index 99d6ea89c4..cdc477b8df 100644 --- a/src/mlpack/bindings/tests/delete_allocated_memory.hpp +++ b/src/mlpack/bindings/tests/delete_allocated_memory.hpp @@ -42,7 +42,7 @@ void DeleteAllocatedMemoryImpl( const typename std::enable_if::value>::type* = 0) { // Delete the allocated memory (hopefully we actually own it). - delete *ANY_CAST(&d.value); + delete *MLPACK_ANY_CAST(&d.value); } template diff --git a/src/mlpack/bindings/tests/get_allocated_memory.hpp b/src/mlpack/bindings/tests/get_allocated_memory.hpp index 4776e9ddb6..f32d0a01f4 100644 --- a/src/mlpack/bindings/tests/get_allocated_memory.hpp +++ b/src/mlpack/bindings/tests/get_allocated_memory.hpp @@ -43,7 +43,7 @@ void* GetAllocatedMemory( const typename std::enable_if::value>::type* = 0) { // Here we have a model; return its memory location. - return *ANY_CAST(&d.value); + return *MLPACK_ANY_CAST(&d.value); } template diff --git a/src/mlpack/bindings/tests/get_param.hpp b/src/mlpack/bindings/tests/get_param.hpp index 6838b2bbc2..715dadc719 100644 --- a/src/mlpack/bindings/tests/get_param.hpp +++ b/src/mlpack/bindings/tests/get_param.hpp @@ -26,7 +26,7 @@ template T& GetParam(util::ParamData& d) { // No mapping is needed, so just cast it directly. - return *ANY_CAST(&d.value); + return *MLPACK_ANY_CAST(&d.value); } /** diff --git a/src/mlpack/bindings/tests/get_printable_param_impl.hpp b/src/mlpack/bindings/tests/get_printable_param_impl.hpp index 23d1820296..ff86de230c 100644 --- a/src/mlpack/bindings/tests/get_printable_param_impl.hpp +++ b/src/mlpack/bindings/tests/get_printable_param_impl.hpp @@ -29,7 +29,7 @@ std::string GetPrintableParam( std::tuple>::value>::type* /* junk */) { std::ostringstream oss; - oss << ANY_CAST(data.value); + oss << MLPACK_ANY_CAST(data.value); return oss.str(); } @@ -39,7 +39,7 @@ std::string GetPrintableParam( util::ParamData& data, const typename std::enable_if::value>::type* /* junk */) { - const T& t = ANY_CAST(data.value); + const T& t = MLPACK_ANY_CAST(data.value); std::ostringstream oss; for (size_t i = 0; i < t.size(); ++i) diff --git a/src/mlpack/bindings/tests/test_option.hpp b/src/mlpack/bindings/tests/test_option.hpp index 8f4bf87c23..ccd64e8f23 100644 --- a/src/mlpack/bindings/tests/test_option.hpp +++ b/src/mlpack/bindings/tests/test_option.hpp @@ -83,7 +83,7 @@ class TestOption data.input = input; data.loaded = false; data.cppType = cppName; - data.value = ANY(defaultValue); + data.value = MLPACK_ANY(defaultValue); const std::string tname = data.tname; diff --git a/src/mlpack/core/data/string_encoding.hpp b/src/mlpack/core/data/string_encoding.hpp index eba8c87849..58e4d2d09d 100644 --- a/src/mlpack/core/data/string_encoding.hpp +++ b/src/mlpack/core/data/string_encoding.hpp @@ -74,7 +74,7 @@ class StringEncoding * @param tokenizer The tokenizer object. * * The tokenization algorithm has to be an object with two public methods: - * 1. operator() which accepts a reference to STRING_VIEW, extracts + * 1. operator() which accepts a reference to MLPACK_STRING_VIEW, extracts * the next token from the given view, removes the prefix containing * the extracted token and returns the token; * 2. IsTokenEmpty() that accepts a token and returns true if the given @@ -108,7 +108,7 @@ class StringEncoding * @param tokenizer The tokenizer object. * * The tokenization algorithm has to be an object with two public methods: - * 1. operator() which accepts a reference to STRING_VIEW, extracts + * 1. operator() which accepts a reference to MLPACK_STRING_VIEW, extracts * the next token from the given view, removes the prefix containing * the extracted token and returns the token; * 2. IsTokenEmpty() that accepts a token and returns true if the given @@ -158,7 +158,7 @@ class StringEncoding * @param policy The policy object. * * The tokenization algorithm has to be an object with two public methods: - * 1. operator() which accepts a reference to STRING_VIEW, extracts + * 1. operator() which accepts a reference to MLPACK_STRING_VIEW, extracts * the next token from the given view, removes the prefix containing * the extracted token and returns the token; * 2. IsTokenEmpty() that accepts a token and returns true if the given @@ -187,7 +187,7 @@ class StringEncoding * @param policy The policy object. * * The tokenization algorithm has to be an object with two public methods: - * 1. operator() which accepts a reference to STRING_VIEW, extracts + * 1. operator() which accepts a reference to MLPACK_STRING_VIEW, extracts * the next token from the given view, removes the prefix containing * the extracted token and returns the token; * 2. IsTokenEmpty() that accepts a token and returns true if the given diff --git a/src/mlpack/core/data/string_encoding_dictionary.hpp b/src/mlpack/core/data/string_encoding_dictionary.hpp index ea429afc01..54ad5f5703 100644 --- a/src/mlpack/core/data/string_encoding_dictionary.hpp +++ b/src/mlpack/core/data/string_encoding_dictionary.hpp @@ -106,20 +106,20 @@ class StringEncodingDictionary }; /* - * Specialization of the StringEncodingDictionary class for STRING_VIEW. + * Specialization of the StringEncodingDictionary class for MLPACK_STRING_VIEW. */ template<> -class StringEncodingDictionary +class StringEncodingDictionary { public: //! A convenient alias for the internal type of the map. using MapType = std::unordered_map< - STRING_VIEW, + MLPACK_STRING_VIEW, size_t, - std::hash>; + std::hash>; //! The type of the token that the dictionary stores. - using TokenType = STRING_VIEW; + using TokenType = MLPACK_STRING_VIEW; //! Construct the default class. StringEncodingDictionary() = default; @@ -156,7 +156,7 @@ class StringEncodingDictionary * * @param token The given token. */ - bool HasToken(const STRING_VIEW token) const + bool HasToken(const MLPACK_STRING_VIEW token) const { return mapping.find(token) != mapping.end(); } @@ -168,7 +168,7 @@ class StringEncodingDictionary * * @param token The given token. */ - size_t AddToken(const STRING_VIEW token) + size_t AddToken(const MLPACK_STRING_VIEW token) { tokens.emplace_back(token); @@ -185,7 +185,7 @@ class StringEncodingDictionary * * @param token The given token. */ - size_t Value(const STRING_VIEW token) const + size_t Value(const MLPACK_STRING_VIEW token) const { return mapping.at(token); } diff --git a/src/mlpack/core/data/string_encoding_impl.hpp b/src/mlpack/core/data/string_encoding_impl.hpp index 06558960e9..71efb0e550 100644 --- a/src/mlpack/core/data/string_encoding_impl.hpp +++ b/src/mlpack/core/data/string_encoding_impl.hpp @@ -66,7 +66,7 @@ void StringEncoding::CreateMap( const std::string& input, const TokenizerType& tokenizer) { - STRING_VIEW strView(input); + MLPACK_STRING_VIEW strView(input); auto token = tokenizer(strView); static_assert( @@ -112,7 +112,7 @@ EncodeHelper(const std::vector& input, // The first pass adds the extracted tokens to the dictionary. for (size_t i = 0; i < input.size(); ++i) { - STRING_VIEW strView(input[i]); + MLPACK_STRING_VIEW strView(input[i]); auto token = tokenizer(strView); static_assert( @@ -143,7 +143,7 @@ EncodeHelper(const std::vector& input, // The second pass writes the encoded values to the output. for (size_t i = 0; i < input.size(); ++i) { - STRING_VIEW strView(input[i]); + MLPACK_STRING_VIEW strView(input[i]); auto token = tokenizer(strView); size_t numTokens = 0; @@ -172,7 +172,7 @@ EncodeHelper(const std::vector& input, // at once. for (size_t i = 0; i < input.size(); ++i) { - STRING_VIEW strView(input[i]); + MLPACK_STRING_VIEW strView(input[i]); auto token = tokenizer(strView); static_assert( diff --git a/src/mlpack/core/data/tokenizers/char_extract.hpp b/src/mlpack/core/data/tokenizers/char_extract.hpp index 084e5e8631..c6912e099b 100644 --- a/src/mlpack/core/data/tokenizers/char_extract.hpp +++ b/src/mlpack/core/data/tokenizers/char_extract.hpp @@ -36,7 +36,7 @@ class CharExtract * * @param str String view to retrieve the next token from. */ - int operator()(STRING_VIEW& str) const + int operator()(MLPACK_STRING_VIEW& str) const { if (str.empty()) return EOF; diff --git a/src/mlpack/core/data/tokenizers/split_by_any_of.hpp b/src/mlpack/core/data/tokenizers/split_by_any_of.hpp index 9097c34f76..d1b8d5e2c9 100644 --- a/src/mlpack/core/data/tokenizers/split_by_any_of.hpp +++ b/src/mlpack/core/data/tokenizers/split_by_any_of.hpp @@ -27,7 +27,7 @@ class SplitByAnyOf { public: //! The type of the token which the tokenizer extracts. - using TokenType = STRING_VIEW; + using TokenType = MLPACK_STRING_VIEW; //! A convenient alias for the mask type. using MaskType = std::array; @@ -37,7 +37,7 @@ class SplitByAnyOf * * @param delimiters The given delimiters. */ - SplitByAnyOf(const STRING_VIEW delimiters) + SplitByAnyOf(const MLPACK_STRING_VIEW delimiters) { mask.fill(false); @@ -51,13 +51,13 @@ class SplitByAnyOf * * @param str String view to retrieve the token from. */ - STRING_VIEW operator()(STRING_VIEW& str) const + MLPACK_STRING_VIEW operator()(MLPACK_STRING_VIEW& str) const { - STRING_VIEW retval; + MLPACK_STRING_VIEW retval; // std::basic_string_view does not have empty function. // Therefore, we are assiging an empty string when reaching the last // delimiter. - STRING_VIEW empty_string{""}; + MLPACK_STRING_VIEW empty_string{""}; while (retval.empty()) { @@ -79,7 +79,7 @@ class SplitByAnyOf * * @param token The given token. */ - static bool IsTokenEmpty(const STRING_VIEW token) + static bool IsTokenEmpty(const MLPACK_STRING_VIEW token) { return token.empty(); } @@ -93,11 +93,11 @@ class SplitByAnyOf /** * The function finds the first character in the given string view equal to * any of the delimiters and returns the position of the character or - * STRING_VIEW::npos if no such character is found. + * MLPACK_STRING_VIEW::npos if no such character is found. * * @param str String where to find the character. */ - size_t FindFirstDelimiter(const STRING_VIEW str) const + size_t FindFirstDelimiter(const MLPACK_STRING_VIEW str) const { for (size_t pos = 0; pos < str.size(); pos++) { diff --git a/src/mlpack/core/util/param_data.hpp b/src/mlpack/core/util/param_data.hpp index 93da599c9b..4477e84de9 100644 --- a/src/mlpack/core/util/param_data.hpp +++ b/src/mlpack/core/util/param_data.hpp @@ -62,7 +62,7 @@ struct ParamData bool loaded; //! The actual value that is held. If the user has passed a different type, //! this may be a tuple containing multiple values. - ANY value; + MLPACK_ANY value; //! The true name of the type, as it would be written in C++. std::string cppType; }; diff --git a/src/mlpack/core/util/params_impl.hpp b/src/mlpack/core/util/params_impl.hpp index 1c53205b94..56a83f2af6 100644 --- a/src/mlpack/core/util/params_impl.hpp +++ b/src/mlpack/core/util/params_impl.hpp @@ -98,7 +98,7 @@ T& Params::Get(const std::string& identifier) } else { - return *ANY_CAST(&d.value); + return *MLPACK_ANY_CAST(&d.value); } } diff --git a/src/mlpack/tests/cli_binding_test.cpp b/src/mlpack/tests/cli_binding_test.cpp index e22dec145e..f5c1048619 100644 --- a/src/mlpack/tests/cli_binding_test.cpp +++ b/src/mlpack/tests/cli_binding_test.cpp @@ -98,7 +98,7 @@ TEST_CASE("GetParamDoubleTest", "[CLIOptionTest]") { util::ParamData d; double x = 5.0; - d.value = ANY(x); + d.value = MLPACK_ANY(x); double* output = NULL; GetParam((util::ParamData&) d, (const void*) NULL, @@ -116,7 +116,7 @@ TEST_CASE("GetParamLoadedMatTest", "[CLIOptionTest]") typedef std::tuple TupleType; TupleType testTuple{filename, 0, 0}; tuple tuple = make_tuple(m, testTuple); - d.value = ANY(tuple); + d.value = MLPACK_ANY(tuple); // Mark it as already loaded. d.input = true; d.loaded = true; @@ -142,7 +142,7 @@ TEST_CASE("GetParamUnloadedMatTest", "[CLIOptionTest]") typedef tuple TupleType; TupleType testTuple{filename, 0, 0}; tuple tuple = make_tuple(m, testTuple); - d.value = ANY(tuple); + d.value = MLPACK_ANY(tuple); // Make sure it is not loaded yet. d.input = true; d.loaded = false; @@ -170,7 +170,7 @@ TEST_CASE("GetParamUmatTest", "[CLIOptionTest]") typedef tuple TupleType; TupleType testTuple{filename, 0, 0}; tuple, TupleType> tuple = make_tuple(m, testTuple); - d.value = ANY(tuple); + d.value = MLPACK_ANY(tuple); // Mark it as already loaded. d.input = true; d.loaded = true; @@ -197,7 +197,7 @@ TEST_CASE("GetParamUnloadedUmatTest", "[CLIOptionTest]") typedef tuple TupleType; TupleType testTuple{filename, 0, 0}; tuple, TupleType> tuple = make_tuple(m, testTuple); - d.value = ANY(tuple); + d.value = MLPACK_ANY(tuple); // Make sure it is not loaded yet. d.input = true; d.loaded = false; @@ -243,7 +243,7 @@ TEST_CASE("GetParamDatasetInfoMatTest", "[CLIOptionTest]") tuple tuple1 = make_tuple(dd, m); tuple tuple2 = make_tuple(tuple1, testTuple); - d.value = ANY(tuple2); + d.value = MLPACK_ANY(tuple2); // Make sure it is not loaded yet. d.input = true; d.loaded = false; @@ -279,7 +279,7 @@ TEST_CASE("GetParamModelTest", "[CLIOptionTest]") // Create tuple. tuple t = make_tuple((GaussianKernel*) NULL, filename); - d.value = ANY(t); + d.value = MLPACK_ANY(t); // Make sure it is not loaded yet. d.input = true; d.loaded = false; @@ -299,7 +299,7 @@ TEST_CASE("RawParamDoubleTest", "[CLIOptionTest]") // This should function the same as GetParam for doubles. util::ParamData d; double x = 5.0; - d.value = ANY(x); + d.value = MLPACK_ANY(x); double* output = NULL; GetParam((util::ParamData&) d, (const void*) NULL, @@ -318,7 +318,7 @@ TEST_CASE("RawParamMatTest", "[CLIOptionTest]") typedef tuple TupleType; TupleType testTuple{filename, 0, 0}; tuple tuple = make_tuple(m, testTuple); - d.value = ANY(tuple); + d.value = MLPACK_ANY(tuple); d.input = true; d.loaded = false; d.noTranspose = false; @@ -343,7 +343,7 @@ TEST_CASE("GetRawParamModelTest", "[CLIOptionTest]") // Create tuple. tuple t = make_tuple(&gk, filename); - d.value = ANY(t); + d.value = MLPACK_ANY(t); // Make sure it is not loaded yet. d.input = true; d.loaded = false; @@ -370,7 +370,7 @@ TEST_CASE("GetRawParamDatasetInfoTest", "[CLIOptionTest]") tuple tuple1 = make_tuple(dd, m); tuple tuple2 = make_tuple(tuple1, testTuple); - d.value = ANY(tuple2); + d.value = MLPACK_ANY(tuple2); // Make sure it is not loaded yet. d.input = true; d.loaded = false; @@ -398,7 +398,7 @@ TEST_CASE("OutputParamMatTest", "[CLIOptionTest]") TupleType testTuple{filename, 0, 0}; tuple t = make_tuple(m, testTuple); - d.value = ANY(t); + d.value = MLPACK_ANY(t); d.input = false; d.noTranspose = false; @@ -426,7 +426,7 @@ TEST_CASE("OutputParamUmatTest", "[CLIOptionTest]") TupleType testTuple{filename, 0, 0}; tuple, TupleType> t = make_tuple(m, testTuple); - d.value = ANY(t); + d.value = MLPACK_ANY(t); d.input = false; d.noTranspose = false; @@ -452,7 +452,7 @@ TEST_CASE("OutputParamModelTest", "[CLIOptionTest]") GaussianKernel gk(5.0); tuple t = make_tuple(&gk, filename); - d.value = ANY(t); + d.value = MLPACK_ANY(t); d.input = false; // Now save it. @@ -474,11 +474,11 @@ TEST_CASE("SetParamDoubleTest", "[CLIOptionTest]") // Create initial value. double dd = 5.0; - d.value = ANY(dd); + d.value = MLPACK_ANY(dd); // Now create second value. double dd2 = 1.0; - ANY a(dd2); + MLPACK_ANY a(dd2); SetParam((util::ParamData&) d, (const void*) &a, (void*) NULL); // Make sure it's the right thing. @@ -496,15 +496,15 @@ TEST_CASE("SetParamBoolTest", "[CLIOptionTest]") // Create initial value. bool b = false; - d.value = ANY(b); + d.value = MLPACK_ANY(b); d.wasPassed = true; // Now create second value. bool b2 = true; - ANY a(b2); + MLPACK_ANY a(b2); SetParam((util::ParamData&) d, (const void*) &a, (void*) NULL); - REQUIRE(ANY_CAST(d.value) == true); + REQUIRE(MLPACK_ANY_CAST(d.value) == true); } // Test that calling SetParam on a matrix sets the string correctly. @@ -517,18 +517,18 @@ TEST_CASE("SetParamMatrixTest", "[CLIOptionTest]") arma::mat m(5, 5, arma::fill::randu); typedef tuple TupleType; TupleType testTuple{filename, 0, 0}; - d.value = ANY(make_tuple(m, testTuple)); + d.value = MLPACK_ANY(make_tuple(m, testTuple)); // Get a new string. string newFilename = "new.csv"; - ANY a2(newFilename); + MLPACK_ANY a2(newFilename); SetParam((util::ParamData&) d, (const void*) &a2, (void*) NULL); // Make sure the change went through. tuple& t = - *ANY_CAST>(&d.value); + *MLPACK_ANY_CAST>(&d.value); REQUIRE(get<0>(get<1>(t)) == "new.csv"); } @@ -540,18 +540,18 @@ TEST_CASE("SetParamModelTest", "[CLIOptionTest]") // Create initial value. string filename = "kernel.bin"; GaussianKernel gk(2.0); - d.value = ANY(make_tuple(&gk, filename)); + d.value = MLPACK_ANY(make_tuple(&gk, filename)); // Get a new string. string newFilename = "new_kernel.bin"; - ANY a2(newFilename); + MLPACK_ANY a2(newFilename); SetParam((util::ParamData&) d, (const void*) &a2, (void*) NULL); // Make sure the change went through. tuple& t = - *ANY_CAST>(&d.value); + *MLPACK_ANY_CAST>(&d.value); REQUIRE(get<1>(t) == "new_kernel.bin"); } @@ -572,19 +572,19 @@ TEST_CASE("SetParamDatasetInfoMatTest", "[CLIOptionTest]") tuple t1 = make_tuple(di, m); tuple, TupleType> t2 = make_tuple(t1, testTuple); - d.value = ANY(t2); + d.value = MLPACK_ANY(t2); d.noTranspose = false; // Now get new filename. string newFilename = "new_filename.csv"; - ANY a2(newFilename); + MLPACK_ANY a2(newFilename); SetParam>((util::ParamData&) d, (const void*) &a2, (void*) NULL); // Check that the name is right. tuple, TupleType>& t3 = - *ANY_CAST, TupleType>>( + *MLPACK_ANY_CAST, TupleType>>( &d.value); REQUIRE(get<0>(get<1>(t3)) == "new_filename.csv"); @@ -597,7 +597,7 @@ TEST_CASE("GetAllocatedMemoryNonModelTest", "[CLIOptionTest]") util::ParamData d; bool b = true; - d.value = ANY(b); + d.value = MLPACK_ANY(b); d.input = true; void* result = (void*) 1; // Invalid pointer, should be overwritten. @@ -613,7 +613,7 @@ TEST_CASE("GetAllocatedMemoryNonModelTest", "[CLIOptionTest]") typedef tuple TupleType; TupleType testTuple{filename, 0, 0}; tuple t = make_tuple(test, testTuple); - d.value = ANY(t); + d.value = MLPACK_ANY(t); result = (void*) 1; @@ -632,7 +632,7 @@ TEST_CASE("GetAllocatedMemoryModelTest", "[CLIOptionTest]") GaussianKernel g(2.0); string filename = "hello.bin"; tuple t = make_tuple(&g, filename); - d.value = ANY(t); + d.value = MLPACK_ANY(t); d.input = true; void* result = NULL; @@ -650,7 +650,7 @@ TEST_CASE("DeleteAllocatedMemoryNonModelTest", "[CLIOptionTest]") util::ParamData d; bool b = true; - d.value = ANY(b); + d.value = MLPACK_ANY(b); d.input = true; DeleteAllocatedMemory((util::ParamData&) d, @@ -661,7 +661,7 @@ TEST_CASE("DeleteAllocatedMemoryNonModelTest", "[CLIOptionTest]") typedef tuple TupleType; TupleType testTuple{filename, 0, 0}; tuple t = make_tuple(test, testTuple); - d.value = ANY(t); + d.value = MLPACK_ANY(t); DeleteAllocatedMemory((util::ParamData&) d, (const void*) NULL, (void*) NULL); @@ -680,7 +680,7 @@ TEST_CASE("DeleteAllocatedMemoryModelTest", "[CLIOptionTest]") string filename = "hello.bin"; tuple t = make_tuple(g, filename); - d.value = ANY(t); + d.value = MLPACK_ANY(t); d.input = false; DeleteAllocatedMemory((util::ParamData&) d, diff --git a/src/mlpack/tests/python_binding_test.cpp b/src/mlpack/tests/python_binding_test.cpp index 5bc1f263b5..1ec07b3ba6 100644 --- a/src/mlpack/tests/python_binding_test.cpp +++ b/src/mlpack/tests/python_binding_test.cpp @@ -87,7 +87,7 @@ TEST_CASE("PyGetParamDoubleTest", "[PythonBindingsTest]") { util::ParamData d; double x = 5.0; - d.value = ANY(x); + d.value = MLPACK_ANY(x); double* output = NULL; GetParam(d, (void*) NULL, (void*) &output); @@ -99,7 +99,7 @@ TEST_CASE("GetParamMatTest", "[PythonBindingsTest]") { util::ParamData d; arma::mat m(5, 5, arma::fill::ones); - d.value = ANY(m); + d.value = MLPACK_ANY(m); arma::mat* output = NULL; GetParam(d, (void*) NULL, (void*) &output); diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index 9c3d84a753..b3299e746f 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -77,7 +77,7 @@ void CheckVectors(const vector>& a, */ TEST_CASE("DictionaryEncodingTest", "[StringEncodingTest]") { - using DictionaryType = StringEncodingDictionary; + using DictionaryType = StringEncodingDictionary; arma::mat output; DictionaryEncoding encoder; @@ -114,7 +114,7 @@ TEST_CASE("DictionaryEncodingTest", "[StringEncodingTest]") */ TEST_CASE("UnicodeDictionaryEncodingTest", "[StringEncodingTest]") { - using DictionaryType = StringEncodingDictionary; + using DictionaryType = StringEncodingDictionary; arma::mat output; DictionaryEncoding encoder; @@ -148,7 +148,7 @@ TEST_CASE("UnicodeDictionaryEncodingTest", "[StringEncodingTest]") */ TEST_CASE("OnePassDictionaryEncodingTest", "[StringEncodingTest]") { - using DictionaryType = StringEncodingDictionary; + using DictionaryType = StringEncodingDictionary; vector> output; DictionaryEncoding encoder( @@ -185,10 +185,10 @@ TEST_CASE("OnePassDictionaryEncodingTest", "[StringEncodingTest]") */ TEST_CASE("SplitByAnyOfTokenizerTest", "[StringEncodingTest]") { - std::vector tokens; - STRING_VIEW line(stringEncodingInput[0]); + std::vector tokens; + MLPACK_STRING_VIEW line(stringEncodingInput[0]); SplitByAnyOf tokenizer(" ,."); - STRING_VIEW token = tokenizer(line); + MLPACK_STRING_VIEW token = tokenizer(line); while (!token.empty()) { @@ -223,10 +223,10 @@ TEST_CASE("SplitByAnyOfTokenizerUnicodeTest", "[StringEncodingTest]") "\xE2\x93\x82\xE2\x93\x81\xE2\x93\x85\xE2\x92\xB6\xE2\x92\xB8\xE2\x93\x80" }; - std::vector tokens; - STRING_VIEW line(stringEncodingUtf8Input[2]); + std::vector tokens; + MLPACK_STRING_VIEW line(stringEncodingUtf8Input[2]); SplitByAnyOf tokenizer(" ,."); - STRING_VIEW token = tokenizer(line); + MLPACK_STRING_VIEW token = tokenizer(line); while (!token.empty()) { tokens.push_back(token); @@ -295,7 +295,7 @@ TEST_CASE("OnePassDictionaryEncodingIndividualCharactersTest", */ TEST_CASE("StringEncodingCopyTest", "[StringEncodingTest]") { - using DictionaryType = StringEncodingDictionary; + using DictionaryType = StringEncodingDictionary; arma::sp_mat output; DictionaryEncoding encoderCopy; SplitByAnyOf tokenizer(" ,."); @@ -331,7 +331,7 @@ TEST_CASE("StringEncodingCopyTest", "[StringEncodingTest]") */ TEST_CASE("StringEncodingMoveTest", "[StringEncodingTest]") { - using DictionaryType = StringEncodingDictionary; + using DictionaryType = StringEncodingDictionary; arma::sp_mat output; DictionaryEncoding encoderCopy; SplitByAnyOf tokenizer(" ,."); @@ -390,21 +390,21 @@ void CheckDictionaries(const StringEncodingDictionary& expected, /** * This is a specialization of the CheckDictionaries() function for - * the STRING_VIEW token type. + * the MLPACK_STRING_VIEW token type. */ template<> void CheckDictionaries( - const StringEncodingDictionary& expected, - const StringEncodingDictionary& obtained) + const StringEncodingDictionary& expected, + const StringEncodingDictionary& obtained) { /* MapType is equal to * - * std::unordered_map>. + * boost::hash>. */ using MapType = - typename StringEncodingDictionary::MapType; + typename StringEncodingDictionary::MapType; const std::deque& expectedTokens = expected.Tokens(); const std::deque& tokens = obtained.Tokens(); @@ -457,9 +457,9 @@ TEST_CASE("StringEncodingDictionarySerialization", "[StringEncodingTest]") for (const string& line : stringEncodingInput) { - STRING_VIEW lineView(line); + MLPACK_STRING_VIEW lineView(line); - STRING_VIEW token = tokenizer(lineView); + MLPACK_STRING_VIEW token = tokenizer(lineView); while (!tokenizer.IsTokenEmpty(token)) { @@ -544,7 +544,7 @@ TEST_CASE("CharExtractDictionaryEncodingSerialization", "[StringEncodingTest]") */ TEST_CASE("BagOfWordsEncodingTest", "[StringEncodingTest]") { - using DictionaryType = StringEncodingDictionary; + using DictionaryType = StringEncodingDictionary; arma::mat output; BagOfWordsEncoding encoder; @@ -619,7 +619,7 @@ TEST_CASE("BagOfWordsEncodingTest", "[StringEncodingTest]") */ TEST_CASE("VectorBagOfWordsEncodingTest", "[StringEncodingTest]") { - using DictionaryType = StringEncodingDictionary; + using DictionaryType = StringEncodingDictionary; vector> output; BagOfWordsEncoding encoder( @@ -713,7 +713,7 @@ TEST_CASE("VectorBagOfWordsEncodingIndividualCharactersTest", */ TEST_CASE("RawCountSmoothIdfEncodingTest", "[StringEncodingTest]") { - using DictionaryType = StringEncodingDictionary; + using DictionaryType = StringEncodingDictionary; arma::mat output; TfIdfEncoding encoder; @@ -815,7 +815,7 @@ TEST_CASE("RawCountSmoothIdfEncodingTest", "[StringEncodingTest]") */ TEST_CASE("VectorRawCountSmoothIdfEncodingTest", "[StringEncodingTest]") { - using DictionaryType = StringEncodingDictionary; + using DictionaryType = StringEncodingDictionary; vector> output; TfIdfEncoding encoder( @@ -974,7 +974,7 @@ TEST_CASE("VectorRawCountSmoothIdfEncodingIndividualCharactersTest", */ TEST_CASE("TfIdfRawCountEncodingTest", "[StringEncodingTest]") { - using DictionaryType = StringEncodingDictionary; + using DictionaryType = StringEncodingDictionary; arma::mat output; TfIdfEncoding encoder( @@ -1024,7 +1024,7 @@ TEST_CASE("TfIdfRawCountEncodingTest", "[StringEncodingTest]") */ TEST_CASE("VectorTfIdfRawCountEncodingTest", "[StringEncodingTest]") { - using DictionaryType = StringEncodingDictionary; + using DictionaryType = StringEncodingDictionary; vector> output; TfIdfEncoding From 66923e8782283d4d56ca66106aeda66c25d82610 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Jun 2022 18:50:06 -0400 Subject: [PATCH 28/39] Remove FORCE_CXX11 option (it is assumed that the user has a C++11 compiler in 2022). --- CMakeLists.txt | 2 -- doc/guide/build.hpp | 3 --- src/mlpack/CMakeLists.txt | 32 ++++++++++++++------------------ 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f4998e69f3..b6f55b75de 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -104,8 +104,6 @@ option(BUILD_MARKDOWN_BINDINGS "Build Markdown bindings for website documentatio option(MATHJAX "Use MathJax for HTML Doxygen output (disabled by default)." OFF) -option(FORCE_CXX11 - "Don't check that the compiler supports C++11, just assume it. Make sure to specify any necessary flag to enable C++11 as part of CXXFLAGS." OFF) option(USE_OPENMP "If available, use OpenMP for parallelization." ON) enable_testing() diff --git a/doc/guide/build.hpp b/doc/guide/build.hpp index 79bf51df64..d992096dab 100644 --- a/doc/guide/build.hpp +++ b/doc/guide/build.hpp @@ -199,9 +199,6 @@ The full list of options mlpack allows: (default ON) - MATHJAX=(ON/OFF): use MathJax for generated Doxygen documentation (default OFF) - - FORCE_CXX11=(ON/OFF): assume that the compiler supports C++11 instead of - checking; be sure to specify any necessary flag to enable C++11 as part - of CXXFLAGS (default OFF) - USE_OPENMP=(ON/OFF): if ON, then use OpenMP if the compiler supports it; if OFF, OpenMP support is manually disabled (default ON) diff --git a/src/mlpack/CMakeLists.txt b/src/mlpack/CMakeLists.txt index 83979dfd20..25aa3856b9 100644 --- a/src/mlpack/CMakeLists.txt +++ b/src/mlpack/CMakeLists.txt @@ -21,24 +21,20 @@ endforeach() # are set in the root CMakeLists.txt. add_library(mlpack ${MLPACK_SRCS}) -# If we are not forcing C++11 support, check that the compiler supports C++11 -# and enable it. -if (NOT FORCE_CXX11) - target_compile_features(mlpack PUBLIC - cxx_decltype - cxx_alias_templates - cxx_auto_type - cxx_lambdas - cxx_constexpr - cxx_rvalue_references - cxx_static_assert - cxx_template_template_parameters - cxx_delegating_constructors - cxx_variadic_templates - cxx_nullptr - cxx_noexcept - ) -endif () +target_compile_features(mlpack PUBLIC + cxx_decltype + cxx_alias_templates + cxx_auto_type + cxx_lambdas + cxx_constexpr + cxx_rvalue_references + cxx_static_assert + cxx_template_template_parameters + cxx_delegating_constructors + cxx_variadic_templates + cxx_nullptr + cxx_noexcept +) # Generate export symbols for Windows, instead of adding __declspec(dllimport) # and __declspec(dllexport) everywhere. However, those modifiers are still From 2fcec249912499cc42653bd6d3d1a137aa840513 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Jun 2022 18:53:56 -0400 Subject: [PATCH 29/39] Bump standard requirement to C++14. --- CMakeLists.txt | 4 ++-- HISTORY.md | 2 ++ src/mlpack/base.hpp | 12 ------------ 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b6f55b75de..5e104244b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,8 +107,8 @@ option(MATHJAX option(USE_OPENMP "If available, use OpenMP for parallelization." ON) enable_testing() -# Set required standard to C++11. -set(CMAKE_CXX_STANDARD 11) +# Set required standard to C++14. +set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED ON) # Ensure that GCC is new enough, if the compiler is GCC. diff --git a/HISTORY.md b/HISTORY.md index 605aaec568..3fe3b827c8 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,7 @@ ### mlpack ?.?.? ###### ????-??-?? + * Bump C++ standard requirement to C++14 (#3233). + * Fix `Perceptron` to work with cross-validation framework (#3190). * Migrate from boost tests to Catch2 framework (#2523), (#2584). diff --git a/src/mlpack/base.hpp b/src/mlpack/base.hpp index e82946f9c9..e5b9e68be8 100644 --- a/src/mlpack/base.hpp +++ b/src/mlpack/base.hpp @@ -65,18 +65,6 @@ #define mlpack_force_inline __forceinline #endif -// Backport this functionality from C++14, if it doesn't exist. -#if __cplusplus <= 201103L -#if !defined(_MSC_VER) || _MSC_VER <= 1800 -namespace std { - -template -using enable_if_t = typename enable_if::type; - -} -#endif -#endif - // Backport std::any from C+17 to C++11 to replace boost::any. // Use mnmlstc backport implementation only if compiler does not // support C++17. From 849aaae98a9794b55a74e5bf126bc5bcf3472dad Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Jun 2022 18:58:47 -0400 Subject: [PATCH 30/39] An extra little sanity check. --- src/mlpack/base.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/base.hpp b/src/mlpack/base.hpp index e5b9e68be8..635e5616e0 100644 --- a/src/mlpack/base.hpp +++ b/src/mlpack/base.hpp @@ -56,6 +56,7 @@ #endif // Give ourselves a nice way to force functions to be inline if we need. +#undef mlpack_force_inline #define mlpack_force_inline #if defined(__GNUG__) && !defined(DEBUG) #undef mlpack_force_inline From 092c9c91cdb4985e2dddd778a6d498f30b6150bc Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Jun 2022 19:21:38 -0400 Subject: [PATCH 31/39] Use MLPACK_USE_OPENMP instead. --- src/mlpack/base.hpp | 7 +++---- src/mlpack/core.hpp | 4 ++-- src/mlpack/core/data/binarize.hpp | 4 ++-- .../bias_svd/bias_svd_function_impl.hpp | 4 ++-- src/mlpack/methods/det/dt_utils_impl.hpp | 4 ++-- src/mlpack/methods/kmeans/kmeans_impl.hpp | 2 +- .../methods/kmeans/naive_kmeans_impl.hpp | 2 +- .../methods/linear_svm/linear_svm_main.cpp | 2 +- src/mlpack/methods/lsh/lsh_search_impl.hpp | 4 ++-- .../random_forest/random_forest_impl.hpp | 6 +++--- .../regularized_svd_function_impl.hpp | 4 ++-- .../async_learning_impl.hpp | 4 ++-- .../svdplusplus/svdplusplus_function_impl.hpp | 4 ++-- src/mlpack/tests/async_learning_test.cpp | 6 +++--- src/mlpack/tests/bias_svd_test.cpp | 2 +- src/mlpack/tests/linear_svm_test.cpp | 2 +- src/mlpack/tests/lsh_test.cpp | 2 +- .../tests/main_tests/linear_svm_test.cpp | 18 +++++++++--------- src/mlpack/tests/regularized_svd_test.cpp | 2 +- src/mlpack/tests/svdplusplus_test.cpp | 2 +- 20 files changed, 42 insertions(+), 43 deletions(-) diff --git a/src/mlpack/base.hpp b/src/mlpack/base.hpp index 635e5616e0..3921b56c98 100644 --- a/src/mlpack/base.hpp +++ b/src/mlpack/base.hpp @@ -100,10 +100,9 @@ // This can be removed with Visual Studio supports an OpenMP version with // unsigned loop variables. -#ifdef _WIN32 - #define omp_size_t intmax_t -#else - #define omp_size_t size_t +#if (defined(_OPENMP) && (_OPENMP >= 201107)) + #undef MLPACK_USE_OPENMP + #define MLPACK_USE_OPENMP #endif // We need to be able to mark functions deprecated. diff --git a/src/mlpack/core.hpp b/src/mlpack/core.hpp index a2275ad188..603ff7bcb9 100644 --- a/src/mlpack/core.hpp +++ b/src/mlpack/core.hpp @@ -111,8 +111,8 @@ #include #include -// Use OpenMP if compiled with -DHAS_OPENMP. -#ifdef HAS_OPENMP +// Use OpenMP if available. +#ifdef MLPACK_USE_OPENMP #include #endif diff --git a/src/mlpack/core/data/binarize.hpp b/src/mlpack/core/data/binarize.hpp index 6602e2d451..c0c87cf4e1 100644 --- a/src/mlpack/core/data/binarize.hpp +++ b/src/mlpack/core/data/binarize.hpp @@ -48,7 +48,7 @@ void Binarize(const arma::Mat& input, T *outPtr = output.memptr(); #pragma omp parallel for - for (omp_size_t i = 0; i < (omp_size_t) input.n_elem; ++i) + for (size_t i = 0; i < (size_t) input.n_elem; ++i) outPtr[i] = inPtr[i] > threshold; } @@ -82,7 +82,7 @@ void Binarize(const arma::Mat& input, output = input; #pragma omp parallel for - for (omp_size_t i = 0; i < (omp_size_t) input.n_cols; ++i) + for (size_t i = 0; i < (size_t) input.n_cols; ++i) output(dimension, i) = input(dimension, i) > threshold; } diff --git a/src/mlpack/methods/bias_svd/bias_svd_function_impl.hpp b/src/mlpack/methods/bias_svd/bias_svd_function_impl.hpp index 004638b442..363f6a199c 100644 --- a/src/mlpack/methods/bias_svd/bias_svd_function_impl.hpp +++ b/src/mlpack/methods/bias_svd/bias_svd_function_impl.hpp @@ -288,7 +288,7 @@ inline double ParallelSGD::Optimize( overallObjective = 0; #pragma omp parallel for reduction(+:overallObjective) - for (omp_size_t j = 0; j < (omp_size_t) function.NumFunctions(); ++j) + for (size_t j = 0; j < (size_t) function.NumFunctions(); ++j) { overallObjective += function.Evaluate(iterate, j); } @@ -324,7 +324,7 @@ inline double ParallelSGD::Optimize( // Each processor gets a subset of the instances. // Each subset is of size threadShareSize. size_t threadId = 0; - #ifdef HAS_OPENMP + #ifdef MLPACK_USE_OPENMP threadId = omp_get_thread_num(); #endif diff --git a/src/mlpack/methods/det/dt_utils_impl.hpp b/src/mlpack/methods/det/dt_utils_impl.hpp index b5c50daa16..612d6f06f7 100644 --- a/src/mlpack/methods/det/dt_utils_impl.hpp +++ b/src/mlpack/methods/det/dt_utils_impl.hpp @@ -183,10 +183,10 @@ DTree* Trainer(MatType& dataset, timers.Start("cross_validation"); // Go through each fold. On the Visual Studio compiler, we have to use // intmax_t because size_t is not yet supported by their OpenMP - // implementation. omp_size_t is the appropriate type according to the + // implementation. size_t is the appropriate type according to the // platform. #pragma omp parallel for shared(prunedSequence, regularizationConstants) - for (omp_size_t fold = 0; fold < (omp_size_t) folds; fold++) + for (size_t fold = 0; fold < (size_t) folds; fold++) { // Break up data into train and test sets. const size_t start = fold * testSize; diff --git a/src/mlpack/methods/kmeans/kmeans_impl.hpp b/src/mlpack/methods/kmeans/kmeans_impl.hpp index e8f782e916..3deb5b2ec9 100644 --- a/src/mlpack/methods/kmeans/kmeans_impl.hpp +++ b/src/mlpack/methods/kmeans/kmeans_impl.hpp @@ -306,7 +306,7 @@ Cluster(const MatType& data, assignments.set_size(data.n_cols); #pragma omp parallel for - for (omp_size_t i = 0; i < (omp_size_t) data.n_cols; ++i) + for (size_t i = 0; i < (size_t) data.n_cols; ++i) { // Find the closest centroid to this point. double minDistance = std::numeric_limits::infinity(); diff --git a/src/mlpack/methods/kmeans/naive_kmeans_impl.hpp b/src/mlpack/methods/kmeans/naive_kmeans_impl.hpp index d829adfbbb..e81fca12c7 100644 --- a/src/mlpack/methods/kmeans/naive_kmeans_impl.hpp +++ b/src/mlpack/methods/kmeans/naive_kmeans_impl.hpp @@ -49,7 +49,7 @@ double NaiveKMeans::Iterate(const arma::mat& centroids, arma::Col localCounts(centroids.n_cols, arma::fill::zeros); #pragma omp for - for (omp_size_t i = 0; i < (omp_size_t) dataset.n_cols; ++i) + for (size_t i = 0; i < (size_t) dataset.n_cols; ++i) { // Find the closest centroid to this point. double minDistance = std::numeric_limits::infinity(); diff --git a/src/mlpack/methods/linear_svm/linear_svm_main.cpp b/src/mlpack/methods/linear_svm/linear_svm_main.cpp index a037ec2271..6adc9a254a 100644 --- a/src/mlpack/methods/linear_svm/linear_svm_main.cpp +++ b/src/mlpack/methods/linear_svm/linear_svm_main.cpp @@ -357,7 +357,7 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers) ens::ConstantStep decayPolicy(stepSize); - #ifdef HAS_OPENMP + #ifdef MLPACK_USE_OPENMP size_t threads = omp_get_max_threads(); #else size_t threads = 1; diff --git a/src/mlpack/methods/lsh/lsh_search_impl.hpp b/src/mlpack/methods/lsh/lsh_search_impl.hpp index ff9b33c351..922c8de2dd 100644 --- a/src/mlpack/methods/lsh/lsh_search_impl.hpp +++ b/src/mlpack/methods/lsh/lsh_search_impl.hpp @@ -908,7 +908,7 @@ void LSHSearch::Search( shared(resultingNeighbors, distances) \ schedule(dynamic)\ reduction(+:avgIndicesReturned) - for (omp_size_t i = 0; i < (omp_size_t) querySet.n_cols; ++i) + for (size_t i = 0; i < (size_t) querySet.n_cols; ++i) { // Go through every query point. // Hash every query into every hash table and eventually into the @@ -970,7 +970,7 @@ Search(const size_t k, shared(resultingNeighbors, distances) \ schedule(dynamic)\ reduction(+:avgIndicesReturned) - for (omp_size_t i = 0; i < (omp_size_t) referenceSet.n_cols; ++i) + for (size_t i = 0; i < (size_t) referenceSet.n_cols; ++i) { // Go through every query point. // Hash every query into every hash table and eventually into the diff --git a/src/mlpack/methods/random_forest/random_forest_impl.hpp b/src/mlpack/methods/random_forest/random_forest_impl.hpp index 93acb3b788..e40b5ee3bd 100644 --- a/src/mlpack/methods/random_forest/random_forest_impl.hpp +++ b/src/mlpack/methods/random_forest/random_forest_impl.hpp @@ -393,7 +393,7 @@ void RandomForest< predictions.set_size(data.n_cols); #pragma omp parallel for - for (omp_size_t i = 0; i < data.n_cols; ++i) + for (size_t i = 0; i < data.n_cols; ++i) { predictions[i] = Classify(data.col(i)); } @@ -430,7 +430,7 @@ void RandomForest< probabilities.set_size(trees[0].NumClasses(), data.n_cols); predictions.set_size(data.n_cols); #pragma omp parallel for - for (omp_size_t i = 0; i < data.n_cols; ++i) + for (size_t i = 0; i < data.n_cols; ++i) { arma::vec probs = probabilities.unsafe_col(i); Classify(data.col(i), predictions[i], probs); @@ -506,7 +506,7 @@ double RandomForest< // Train each tree individually. #pragma omp parallel for reduction( + : totalGain) - for (omp_size_t i = 0; i < numTrees; ++i) + for (size_t i = 0; i < numTrees; ++i) { MatType bootstrapDataset; arma::Row bootstrapLabels; diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp b/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp index 1a3ec638be..0d0205ed69 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp @@ -243,7 +243,7 @@ inline double ParallelSGD::Optimize( overallObjective = 0; #pragma omp parallel for reduction(+:overallObjective) - for (omp_size_t j = 0; j < (omp_size_t) function.NumFunctions(); ++j) + for (size_t j = 0; j < (size_t) function.NumFunctions(); ++j) { overallObjective += function.Evaluate(iterate, j); } @@ -279,7 +279,7 @@ inline double ParallelSGD::Optimize( // Each processor gets a subset of the instances. // Each subset is of size threadShareSize. size_t threadId = 0; - #ifdef HAS_OPENMP + #ifdef MLPACK_USE_OPENMP threadId = omp_get_thread_num(); #endif diff --git a/src/mlpack/methods/reinforcement_learning/async_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/async_learning_impl.hpp index c67e9beaa7..daff6a4ae5 100644 --- a/src/mlpack/methods/reinforcement_learning/async_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/async_learning_impl.hpp @@ -97,11 +97,11 @@ void AsyncLearning< #pragma omp parallel for shared(stop, workers, tasks, learningNetwork, \ targetNetwork, totalSteps, policy) - for (omp_size_t i = 0; i < numThreads; ++i) + for (size_t i = 0; i < numThreads; ++i) { #pragma omp critical { - #ifdef HAS_OPENMP + #ifdef MLPACK_USE_OPENMP Log::Debug << "Thread " << omp_get_thread_num() << " started." << std::endl; #endif diff --git a/src/mlpack/methods/svdplusplus/svdplusplus_function_impl.hpp b/src/mlpack/methods/svdplusplus/svdplusplus_function_impl.hpp index d575848902..f991c03b33 100644 --- a/src/mlpack/methods/svdplusplus/svdplusplus_function_impl.hpp +++ b/src/mlpack/methods/svdplusplus/svdplusplus_function_impl.hpp @@ -425,7 +425,7 @@ inline double ParallelSGD::Optimize( overallObjective = 0; #pragma omp parallel for reduction(+:overallObjective) - for (omp_size_t j = 0; j < (omp_size_t) function.NumFunctions(); ++j) + for (size_t j = 0; j < (size_t) function.NumFunctions(); ++j) { overallObjective += function.Evaluate(iterate, j); } @@ -461,7 +461,7 @@ inline double ParallelSGD::Optimize( // Each processor gets a subset of the instances. // Each subset is of size threadShareSize. size_t threadId = 0; - #ifdef HAS_OPENMP + #ifdef MLPACK_USE_OPENMP threadId = omp_get_thread_num(); #endif diff --git a/src/mlpack/tests/async_learning_test.cpp b/src/mlpack/tests/async_learning_test.cpp index d788386c4f..b45a96aa52 100644 --- a/src/mlpack/tests/async_learning_test.cpp +++ b/src/mlpack/tests/async_learning_test.cpp @@ -38,7 +38,7 @@ TEST_CASE("OneStepQLearningTest", "[AsyncLearningTest]") * This is for the Travis CI server, in your own machine you should use more * threads. */ - #ifdef HAS_OPENMP + #ifdef MLPACK_USE_OPENMP omp_set_num_threads(1); #endif @@ -114,7 +114,7 @@ TEST_CASE("OneStepSarsaTest", "[AsyncLearningTest]") * This is for the Travis CI server, in your own machine you shuold use more * threads. */ - #ifdef HAS_OPENMP + #ifdef MLPACK_USE_OPENMP omp_set_num_threads(1); #endif @@ -192,7 +192,7 @@ TEST_CASE("NStepQLearningTest", "[AsyncLearningTest]") * This is for the Travis CI server, in your own machine you shuold use more * threads. */ - #ifdef HAS_OPENMP + #ifdef MLPACK_USE_OPENMP omp_set_num_threads(1); #endif diff --git a/src/mlpack/tests/bias_svd_test.cpp b/src/mlpack/tests/bias_svd_test.cpp index bb9f3936f3..3511c2ee3b 100644 --- a/src/mlpack/tests/bias_svd_test.cpp +++ b/src/mlpack/tests/bias_svd_test.cpp @@ -301,7 +301,7 @@ TEST_CASE("BiasSVDFunctionOptimize", "[BiasSVDTest]") // The test is only compiled if the user has specified OpenMP to be // used. -#ifdef HAS_OPENMP +#ifdef MLPACK_USE_OPENMP // Test Bias SVD with parallel SGD. TEST_CASE("BiasSVDFunctionParallelOptimize", "[BiasSVDTest]") diff --git a/src/mlpack/tests/linear_svm_test.cpp b/src/mlpack/tests/linear_svm_test.cpp index 8bfbe64e2b..c75a3d70da 100644 --- a/src/mlpack/tests/linear_svm_test.cpp +++ b/src/mlpack/tests/linear_svm_test.cpp @@ -748,7 +748,7 @@ TEST_CASE("LinearSVMDeltaLBFGSTwoClasses", "[LinearSVMTest]") * The test is only compiled if the user has specified OpenMP to be * used. */ -#ifdef HAS_OPENMP +#ifdef MLPACK_USE_OPENMP /** * Test training of linear svm on a simple dataset using diff --git a/src/mlpack/tests/lsh_test.cpp b/src/mlpack/tests/lsh_test.cpp index 5328609dab..c6d91c63ca 100644 --- a/src/mlpack/tests/lsh_test.cpp +++ b/src/mlpack/tests/lsh_test.cpp @@ -768,7 +768,7 @@ TEST_CASE("LSHTestEmptyConstructorTest", "[LSHTest]") // These two tests are only compiled if the user has specified OpenMP to be // used. -#ifdef HAS_OPENMP +#ifdef MLPACK_USE_OPENMP /** * Test: This test verifies that parallel query processing returns correct * results for the bichromatic search. diff --git a/src/mlpack/tests/main_tests/linear_svm_test.cpp b/src/mlpack/tests/main_tests/linear_svm_test.cpp index 7b32f158f4..6ff5a69da1 100644 --- a/src/mlpack/tests/main_tests/linear_svm_test.cpp +++ b/src/mlpack/tests/main_tests/linear_svm_test.cpp @@ -654,7 +654,7 @@ TEST_CASE_METHOD(LinearSVMTestFixture, "LinearSVMDiffInterceptTestWithPsgd", SetInputParam("optimizer", std::string("psgd")); // First solution. - #ifdef HAS_OPENMP + #ifdef MLPACK_USE_OPENMP omp_set_num_threads(1); #endif @@ -675,7 +675,7 @@ TEST_CASE_METHOD(LinearSVMTestFixture, "LinearSVMDiffInterceptTestWithPsgd", SetInputParam("no_intercept", bool(true)); // Second solution. - #ifdef HAS_OPENMP + #ifdef MLPACK_USE_OPENMP omp_set_num_threads(1); #endif @@ -735,7 +735,7 @@ TEST_CASE_METHOD(LinearSVMTestFixture, "LinearSVMDiffEpochsTest", SetInputParam("epochs", int(5)); // First solution. - #ifdef HAS_OPENMP + #ifdef MLPACK_USE_OPENMP omp_set_num_threads(1); #endif @@ -756,7 +756,7 @@ TEST_CASE_METHOD(LinearSVMTestFixture, "LinearSVMDiffEpochsTest", SetInputParam("epochs", int(10)); // Second solution. - #ifdef HAS_OPENMP + #ifdef MLPACK_USE_OPENMP omp_set_num_threads(1); #endif @@ -792,7 +792,7 @@ TEST_CASE_METHOD(LinearSVMTestFixture, "LinearSVMDiffStepSizeTest", SetInputParam("step_size", double(0.02)); // First solution. - #ifdef HAS_OPENMP + #ifdef MLPACK_USE_OPENMP omp_set_num_threads(1); #endif @@ -814,7 +814,7 @@ TEST_CASE_METHOD(LinearSVMTestFixture, "LinearSVMDiffStepSizeTest", SetInputParam("step_size", double(1.02)); // Second solution. - #ifdef HAS_OPENMP + #ifdef MLPACK_USE_OPENMP omp_set_num_threads(1); #endif @@ -850,7 +850,7 @@ TEST_CASE_METHOD(LinearSVMTestFixture, "LinearSVMDiffToleranceTest", SetInputParam("tolerance", double(1e-1)); // First solution. - #ifdef HAS_OPENMP + #ifdef MLPACK_USE_OPENMP omp_set_num_threads(1); #endif @@ -872,7 +872,7 @@ TEST_CASE_METHOD(LinearSVMTestFixture, "LinearSVMDiffToleranceTest", SetInputParam("tolerance", double(1e-10)); // Second solution. - #ifdef HAS_OPENMP + #ifdef MLPACK_USE_OPENMP omp_set_num_threads(1); #endif @@ -922,7 +922,7 @@ TEST_CASE_METHOD(LinearSVMTestFixture, "LinearSVMDiffOptimizerTest", SetInputParam("optimizer", std::string("psgd")); // Second solution. - #ifdef HAS_OPENMP + #ifdef MLPACK_USE_OPENMP omp_set_num_threads(1); #endif diff --git a/src/mlpack/tests/regularized_svd_test.cpp b/src/mlpack/tests/regularized_svd_test.cpp index eb412dcd26..73909fcbc5 100644 --- a/src/mlpack/tests/regularized_svd_test.cpp +++ b/src/mlpack/tests/regularized_svd_test.cpp @@ -251,7 +251,7 @@ TEST_CASE("RegularizedSVDFunctionOptimize", "[RegularizedSVDTest]") // The test is only compiled if the user has specified OpenMP to be // used. -#ifdef HAS_OPENMP +#ifdef MLPACK_USE_OPENMP // Test Regularized SVD with parallel SGD. TEST_CASE("RegularizedSVDFunctionOptimizeHOGWILD", "[RegularizedSVDTest]") diff --git a/src/mlpack/tests/svdplusplus_test.cpp b/src/mlpack/tests/svdplusplus_test.cpp index 4ca13e55ad..1607c1e065 100644 --- a/src/mlpack/tests/svdplusplus_test.cpp +++ b/src/mlpack/tests/svdplusplus_test.cpp @@ -437,7 +437,7 @@ TEST_CASE("SVDPlusPlusFunctionOptimize", "[SVDPlusPlusTest]") // The test is only compiled if the user has specified OpenMP to be // used. -#ifdef HAS_OPENMP +#ifdef MLPACK_USE_OPENMP // Test SVDPlusPlus with parallel SGD. TEST_CASE("SVDPlusPlusFunctionParallelOptimize", "[SVDPlusPlusTest]") From 707e2b0a860c88b1091e1b65bd54e1ba4b831479 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 6 Jul 2022 20:49:08 -0400 Subject: [PATCH 32/39] Fix typo. --- src/mlpack/base.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/base.hpp b/src/mlpack/base.hpp index 3921b56c98..8700ea2d37 100644 --- a/src/mlpack/base.hpp +++ b/src/mlpack/base.hpp @@ -98,7 +98,7 @@ // defines. #include -// This can be removed with Visual Studio supports an OpenMP version with +// This can be removed when Visual Studio supports an OpenMP version with // unsigned loop variables. #if (defined(_OPENMP) && (_OPENMP >= 201107)) #undef MLPACK_USE_OPENMP From b6b612391036c5a3b0c8f3af65085f6b90bdd791 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 6 Jul 2022 20:49:46 -0400 Subject: [PATCH 33/39] Remove outdated comment. --- src/mlpack/methods/det/dt_utils_impl.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/mlpack/methods/det/dt_utils_impl.hpp b/src/mlpack/methods/det/dt_utils_impl.hpp index 612d6f06f7..def663fdfd 100644 --- a/src/mlpack/methods/det/dt_utils_impl.hpp +++ b/src/mlpack/methods/det/dt_utils_impl.hpp @@ -181,10 +181,7 @@ DTree* Trainer(MatType& dataset, regularizationConstants.fill(0.0); timers.Start("cross_validation"); - // Go through each fold. On the Visual Studio compiler, we have to use - // intmax_t because size_t is not yet supported by their OpenMP - // implementation. size_t is the appropriate type according to the - // platform. + // Go through each fold. #pragma omp parallel for shared(prunedSequence, regularizationConstants) for (size_t fold = 0; fold < (size_t) folds; fold++) { From 91b493f4e4a7ec4c8127ccf92356063e1d301786 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 7 Jul 2022 22:47:47 -0400 Subject: [PATCH 34/39] Pass the argument directly to an MLPACK_ANY, instead of wrapping it. This fixes the failing tests (or it should). --- src/mlpack/bindings/R/R_option.hpp | 2 +- src/mlpack/bindings/cli/cli_option.hpp | 4 +- src/mlpack/bindings/cli/set_param.hpp | 2 +- src/mlpack/bindings/go/go_option.hpp | 2 +- src/mlpack/bindings/julia/julia_option.hpp | 2 +- src/mlpack/bindings/markdown/md_option.hpp | 2 +- src/mlpack/bindings/python/py_option.hpp | 2 +- src/mlpack/bindings/tests/test_option.hpp | 2 +- src/mlpack/tests/cli_binding_test.cpp | 60 +++++++++++----------- src/mlpack/tests/python_binding_test.cpp | 4 +- 10 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/mlpack/bindings/R/R_option.hpp b/src/mlpack/bindings/R/R_option.hpp index a946052836..f03626800a 100644 --- a/src/mlpack/bindings/R/R_option.hpp +++ b/src/mlpack/bindings/R/R_option.hpp @@ -73,7 +73,7 @@ class ROption data.cppType = cppName; // Every parameter we'll get from R will have the correct type. - data.value = MLPACK_ANY(defaultValue); + data.value = defaultValue; // Set the function pointers that we'll need. All of these function // pointers will be used by both the program that generates the R, and diff --git a/src/mlpack/bindings/cli/cli_option.hpp b/src/mlpack/bindings/cli/cli_option.hpp index ee041c0c22..52c41474ce 100644 --- a/src/mlpack/bindings/cli/cli_option.hpp +++ b/src/mlpack/bindings/cli/cli_option.hpp @@ -95,12 +95,12 @@ class CLIOption typename ParameterType::type>::type>::value) { - data.value = MLPACK_ANY(defaultValue); + data.value = defaultValue; } else { typename ParameterType::type>::type tmp; - data.value = MLPACK_ANY(std::tuple(defaultValue, tmp)); + data.value = std::tuple(defaultValue, tmp); } const std::string tname = data.tname; diff --git a/src/mlpack/bindings/cli/set_param.hpp b/src/mlpack/bindings/cli/set_param.hpp index d575896c8d..5d80b07d36 100644 --- a/src/mlpack/bindings/cli/set_param.hpp +++ b/src/mlpack/bindings/cli/set_param.hpp @@ -34,7 +34,7 @@ void SetParam( const typename std::enable_if::value>::type* = 0) { // No mapping is needed. - d.value = value; + d.value = *MLPACK_ANY_CAST(&value); } /** diff --git a/src/mlpack/bindings/go/go_option.hpp b/src/mlpack/bindings/go/go_option.hpp index 5290e2738c..000db2c9b0 100644 --- a/src/mlpack/bindings/go/go_option.hpp +++ b/src/mlpack/bindings/go/go_option.hpp @@ -78,7 +78,7 @@ class GoOption data.loaded = false; data.cppType = cppName; - data.value = MLPACK_ANY(defaultValue); + data.value = defaultValue; // Set the function pointers that we'll need. All of these function // pointers will be used by both the program that generates the .cpp, diff --git a/src/mlpack/bindings/julia/julia_option.hpp b/src/mlpack/bindings/julia/julia_option.hpp index d8adeb13e3..b1bcaf66ec 100644 --- a/src/mlpack/bindings/julia/julia_option.hpp +++ b/src/mlpack/bindings/julia/julia_option.hpp @@ -64,7 +64,7 @@ class JuliaOption data.cppType = cppName; // Every parameter we'll get from Julia will have the correct type. - data.value = MLPACK_ANY(defaultValue); + data.value = defaultValue; // Set the function pointers that we'll need. All of these function // pointers will be used by both the program that generates the pyx, and diff --git a/src/mlpack/bindings/markdown/md_option.hpp b/src/mlpack/bindings/markdown/md_option.hpp index aecbdb785e..1a502a64de 100644 --- a/src/mlpack/bindings/markdown/md_option.hpp +++ b/src/mlpack/bindings/markdown/md_option.hpp @@ -63,7 +63,7 @@ class MDOption data.cppType = cppName; // Every parameter we'll get from Markdown will have the correct type. - data.value = MLPACK_ANY(defaultValue); + data.value = defaultValue; // Set the function pointers that we'll need. Most of these simply delegate // to the current binding type's implementation. Any new language will need diff --git a/src/mlpack/bindings/python/py_option.hpp b/src/mlpack/bindings/python/py_option.hpp index 5c92eeee7c..ac6badfc9f 100644 --- a/src/mlpack/bindings/python/py_option.hpp +++ b/src/mlpack/bindings/python/py_option.hpp @@ -64,7 +64,7 @@ class PyOption data.cppType = cppName; // Every parameter we'll get from Python will have the correct type. - data.value = MLPACK_ANY(defaultValue); + data.value = defaultValue; // Set the function pointers that we'll need. All of these function // pointers will be used by both the program that generates the pyx, and diff --git a/src/mlpack/bindings/tests/test_option.hpp b/src/mlpack/bindings/tests/test_option.hpp index ccd64e8f23..28681c7fc3 100644 --- a/src/mlpack/bindings/tests/test_option.hpp +++ b/src/mlpack/bindings/tests/test_option.hpp @@ -83,7 +83,7 @@ class TestOption data.input = input; data.loaded = false; data.cppType = cppName; - data.value = MLPACK_ANY(defaultValue); + data.value = defaultValue; const std::string tname = data.tname; diff --git a/src/mlpack/tests/cli_binding_test.cpp b/src/mlpack/tests/cli_binding_test.cpp index f5c1048619..42659e463a 100644 --- a/src/mlpack/tests/cli_binding_test.cpp +++ b/src/mlpack/tests/cli_binding_test.cpp @@ -98,7 +98,7 @@ TEST_CASE("GetParamDoubleTest", "[CLIOptionTest]") { util::ParamData d; double x = 5.0; - d.value = MLPACK_ANY(x); + d.value = x; double* output = NULL; GetParam((util::ParamData&) d, (const void*) NULL, @@ -116,7 +116,7 @@ TEST_CASE("GetParamLoadedMatTest", "[CLIOptionTest]") typedef std::tuple TupleType; TupleType testTuple{filename, 0, 0}; tuple tuple = make_tuple(m, testTuple); - d.value = MLPACK_ANY(tuple); + d.value = tuple; // Mark it as already loaded. d.input = true; d.loaded = true; @@ -142,7 +142,7 @@ TEST_CASE("GetParamUnloadedMatTest", "[CLIOptionTest]") typedef tuple TupleType; TupleType testTuple{filename, 0, 0}; tuple tuple = make_tuple(m, testTuple); - d.value = MLPACK_ANY(tuple); + d.value = tuple; // Make sure it is not loaded yet. d.input = true; d.loaded = false; @@ -170,7 +170,7 @@ TEST_CASE("GetParamUmatTest", "[CLIOptionTest]") typedef tuple TupleType; TupleType testTuple{filename, 0, 0}; tuple, TupleType> tuple = make_tuple(m, testTuple); - d.value = MLPACK_ANY(tuple); + d.value = tuple; // Mark it as already loaded. d.input = true; d.loaded = true; @@ -197,7 +197,7 @@ TEST_CASE("GetParamUnloadedUmatTest", "[CLIOptionTest]") typedef tuple TupleType; TupleType testTuple{filename, 0, 0}; tuple, TupleType> tuple = make_tuple(m, testTuple); - d.value = MLPACK_ANY(tuple); + d.value = tuple; // Make sure it is not loaded yet. d.input = true; d.loaded = false; @@ -243,7 +243,7 @@ TEST_CASE("GetParamDatasetInfoMatTest", "[CLIOptionTest]") tuple tuple1 = make_tuple(dd, m); tuple tuple2 = make_tuple(tuple1, testTuple); - d.value = MLPACK_ANY(tuple2); + d.value = tuple2; // Make sure it is not loaded yet. d.input = true; d.loaded = false; @@ -279,7 +279,7 @@ TEST_CASE("GetParamModelTest", "[CLIOptionTest]") // Create tuple. tuple t = make_tuple((GaussianKernel*) NULL, filename); - d.value = MLPACK_ANY(t); + d.value = t; // Make sure it is not loaded yet. d.input = true; d.loaded = false; @@ -299,7 +299,7 @@ TEST_CASE("RawParamDoubleTest", "[CLIOptionTest]") // This should function the same as GetParam for doubles. util::ParamData d; double x = 5.0; - d.value = MLPACK_ANY(x); + d.value = x; double* output = NULL; GetParam((util::ParamData&) d, (const void*) NULL, @@ -318,7 +318,7 @@ TEST_CASE("RawParamMatTest", "[CLIOptionTest]") typedef tuple TupleType; TupleType testTuple{filename, 0, 0}; tuple tuple = make_tuple(m, testTuple); - d.value = MLPACK_ANY(tuple); + d.value = tuple; d.input = true; d.loaded = false; d.noTranspose = false; @@ -343,7 +343,7 @@ TEST_CASE("GetRawParamModelTest", "[CLIOptionTest]") // Create tuple. tuple t = make_tuple(&gk, filename); - d.value = MLPACK_ANY(t); + d.value = t; // Make sure it is not loaded yet. d.input = true; d.loaded = false; @@ -370,7 +370,7 @@ TEST_CASE("GetRawParamDatasetInfoTest", "[CLIOptionTest]") tuple tuple1 = make_tuple(dd, m); tuple tuple2 = make_tuple(tuple1, testTuple); - d.value = MLPACK_ANY(tuple2); + d.value = tuple2; // Make sure it is not loaded yet. d.input = true; d.loaded = false; @@ -398,7 +398,7 @@ TEST_CASE("OutputParamMatTest", "[CLIOptionTest]") TupleType testTuple{filename, 0, 0}; tuple t = make_tuple(m, testTuple); - d.value = MLPACK_ANY(t); + d.value = t; d.input = false; d.noTranspose = false; @@ -426,7 +426,7 @@ TEST_CASE("OutputParamUmatTest", "[CLIOptionTest]") TupleType testTuple{filename, 0, 0}; tuple, TupleType> t = make_tuple(m, testTuple); - d.value = MLPACK_ANY(t); + d.value = t; d.input = false; d.noTranspose = false; @@ -452,7 +452,7 @@ TEST_CASE("OutputParamModelTest", "[CLIOptionTest]") GaussianKernel gk(5.0); tuple t = make_tuple(&gk, filename); - d.value = MLPACK_ANY(t); + d.value = t; d.input = false; // Now save it. @@ -474,11 +474,11 @@ TEST_CASE("SetParamDoubleTest", "[CLIOptionTest]") // Create initial value. double dd = 5.0; - d.value = MLPACK_ANY(dd); + d.value = dd; // Now create second value. double dd2 = 1.0; - MLPACK_ANY a(dd2); + MLPACK_ANY a = dd2; SetParam((util::ParamData&) d, (const void*) &a, (void*) NULL); // Make sure it's the right thing. @@ -496,12 +496,12 @@ TEST_CASE("SetParamBoolTest", "[CLIOptionTest]") // Create initial value. bool b = false; - d.value = MLPACK_ANY(b); + d.value = b; d.wasPassed = true; // Now create second value. bool b2 = true; - MLPACK_ANY a(b2); + MLPACK_ANY a = b2; SetParam((util::ParamData&) d, (const void*) &a, (void*) NULL); REQUIRE(MLPACK_ANY_CAST(d.value) == true); @@ -517,11 +517,11 @@ TEST_CASE("SetParamMatrixTest", "[CLIOptionTest]") arma::mat m(5, 5, arma::fill::randu); typedef tuple TupleType; TupleType testTuple{filename, 0, 0}; - d.value = MLPACK_ANY(make_tuple(m, testTuple)); + d.value = make_tuple(m, testTuple); // Get a new string. string newFilename = "new.csv"; - MLPACK_ANY a2(newFilename); + MLPACK_ANY a2 = newFilename; SetParam((util::ParamData&) d, (const void*) &a2, (void*) NULL); @@ -540,11 +540,11 @@ TEST_CASE("SetParamModelTest", "[CLIOptionTest]") // Create initial value. string filename = "kernel.bin"; GaussianKernel gk(2.0); - d.value = MLPACK_ANY(make_tuple(&gk, filename)); + d.value = make_tuple(&gk, filename); // Get a new string. string newFilename = "new_kernel.bin"; - MLPACK_ANY a2(newFilename); + MLPACK_ANY a2 = newFilename; SetParam((util::ParamData&) d, (const void*) &a2, (void*) NULL); @@ -572,12 +572,12 @@ TEST_CASE("SetParamDatasetInfoMatTest", "[CLIOptionTest]") tuple t1 = make_tuple(di, m); tuple, TupleType> t2 = make_tuple(t1, testTuple); - d.value = MLPACK_ANY(t2); + d.value = t2; d.noTranspose = false; // Now get new filename. string newFilename = "new_filename.csv"; - MLPACK_ANY a2(newFilename); + MLPACK_ANY a2 = newFilename; SetParam>((util::ParamData&) d, (const void*) &a2, (void*) NULL); @@ -597,7 +597,7 @@ TEST_CASE("GetAllocatedMemoryNonModelTest", "[CLIOptionTest]") util::ParamData d; bool b = true; - d.value = MLPACK_ANY(b); + d.value = b; d.input = true; void* result = (void*) 1; // Invalid pointer, should be overwritten. @@ -613,7 +613,7 @@ TEST_CASE("GetAllocatedMemoryNonModelTest", "[CLIOptionTest]") typedef tuple TupleType; TupleType testTuple{filename, 0, 0}; tuple t = make_tuple(test, testTuple); - d.value = MLPACK_ANY(t); + d.value = t; result = (void*) 1; @@ -632,7 +632,7 @@ TEST_CASE("GetAllocatedMemoryModelTest", "[CLIOptionTest]") GaussianKernel g(2.0); string filename = "hello.bin"; tuple t = make_tuple(&g, filename); - d.value = MLPACK_ANY(t); + d.value = t; d.input = true; void* result = NULL; @@ -650,7 +650,7 @@ TEST_CASE("DeleteAllocatedMemoryNonModelTest", "[CLIOptionTest]") util::ParamData d; bool b = true; - d.value = MLPACK_ANY(b); + d.value = b; d.input = true; DeleteAllocatedMemory((util::ParamData&) d, @@ -661,7 +661,7 @@ TEST_CASE("DeleteAllocatedMemoryNonModelTest", "[CLIOptionTest]") typedef tuple TupleType; TupleType testTuple{filename, 0, 0}; tuple t = make_tuple(test, testTuple); - d.value = MLPACK_ANY(t); + d.value = t; DeleteAllocatedMemory((util::ParamData&) d, (const void*) NULL, (void*) NULL); @@ -680,7 +680,7 @@ TEST_CASE("DeleteAllocatedMemoryModelTest", "[CLIOptionTest]") string filename = "hello.bin"; tuple t = make_tuple(g, filename); - d.value = MLPACK_ANY(t); + d.value = t; d.input = false; DeleteAllocatedMemory((util::ParamData&) d, diff --git a/src/mlpack/tests/python_binding_test.cpp b/src/mlpack/tests/python_binding_test.cpp index 1ec07b3ba6..6914af52c0 100644 --- a/src/mlpack/tests/python_binding_test.cpp +++ b/src/mlpack/tests/python_binding_test.cpp @@ -87,7 +87,7 @@ TEST_CASE("PyGetParamDoubleTest", "[PythonBindingsTest]") { util::ParamData d; double x = 5.0; - d.value = MLPACK_ANY(x); + d.value = x; double* output = NULL; GetParam(d, (void*) NULL, (void*) &output); @@ -99,7 +99,7 @@ TEST_CASE("GetParamMatTest", "[PythonBindingsTest]") { util::ParamData d; arma::mat m(5, 5, arma::fill::ones); - d.value = MLPACK_ANY(m); + d.value = m; arma::mat* output = NULL; GetParam(d, (void*) NULL, (void*) &output); From 5268f559331afad050c15ca252f3aaf6d86f0079 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 11 Jul 2022 11:57:29 -0400 Subject: [PATCH 35/39] Clean up OpenMP detection. --- CMakeLists.txt | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5e104244b4..75ee5781cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -373,26 +373,16 @@ endif() set(MLPACK_INCLUDE_DIRS ${MLPACK_INCLUDE_DIRS} ${CEREAL_INCLUDE_DIR}) # Detect OpenMP support in a compiler. If the compiler supports OpenMP, flags -# to compile with OpenMP are returned and added and the HAS_OPENMP definition -# is added for compilation. -# -# This way we can skip calls to functions defined in omp.h with code like: -# #ifdef HAS_OPENMP -# { -# ... openMP code here ... -# } -# #endif +# to compile with OpenMP are returned and added. Note that MSVC does not +# support a new-enough version of OpenMP to be useful. if (USE_OPENMP) find_package(OpenMP) endif () -if (OPENMP_FOUND) - add_definitions(-DHAS_OPENMP) +if (OpenMP_FOUND AND OpenMP_CXX_VERSION VERSION_GREATER_EQUAL 3.0.0) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") - if (OpenMP_CXX_FOUND) - set(MLPACK_LIBRARIES ${MLPACK_LIBRARIES} ${OpenMP_CXX_LIBRARIES}) - endif () + set(MLPACK_LIBRARIES ${MLPACK_LIBRARIES} ${OpenMP_CXX_LIBRARIES}) else () # Disable warnings for all the unknown OpenMP pragmas. if (NOT MSVC) From d21439b17ac4bde4d69b97c659fb02528094ee6f Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 12 Jul 2022 08:05:52 -0400 Subject: [PATCH 36/39] Bump minimum requirement for bindings to C++14. --- src/mlpack/bindings/R/mlpack/DESCRIPTION.in | 3 +-- src/mlpack/bindings/python/setup.py.in | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/mlpack/bindings/R/mlpack/DESCRIPTION.in b/src/mlpack/bindings/R/mlpack/DESCRIPTION.in index 9ccc0e0e03..4745c42a17 100644 --- a/src/mlpack/bindings/R/mlpack/DESCRIPTION.in +++ b/src/mlpack/bindings/R/mlpack/DESCRIPTION.in @@ -7,8 +7,7 @@ Description: A fast, flexible machine learning library, written in C++, that aims to provide fast, extensible implementations of cutting-edge machine learning algorithms. See also Curtin et al. (2018) . -SystemRequirements: A C++11 compiler. Versions 4.8.*, 4.9.* or later of GCC - will be fine. +SystemRequirements: A C++14 compiler. Version 5 or later of GCC will be fine. License: BSD_3_clause + file LICENSE Depends: R (>= 4.0.0) Imports: Rcpp (>= 0.12.12) diff --git a/src/mlpack/bindings/python/setup.py.in b/src/mlpack/bindings/python/setup.py.in index 0a0efd8318..492a89183f 100644 --- a/src/mlpack/bindings/python/setup.py.in +++ b/src/mlpack/bindings/python/setup.py.in @@ -52,7 +52,7 @@ if os.getenv('NO_BUILD') == '1': else: cxx_flags = '${CMAKE_CXX_FLAGS}'.strip() cxx_flags = re.sub(' +', ' ', cxx_flags) - extra_args = ['-DBINDING_TYPE=BINDING_TYPE_PYX', '-std=c++11'] + extra_args = ['-DBINDING_TYPE=BINDING_TYPE_PYX', '-std=c++14'] if '${OpenMP_CXX_FLAGS}' != '': extra_args.append('${OpenMP_CXX_FLAGS}') if cxx_flags: From ab7fd488a9aabdfa7ccf57193f23f56c49a1b0b5 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sat, 30 Jul 2022 05:13:20 -0400 Subject: [PATCH 37/39] Some reordering to help the static analysis job (hopefully!). --- src/mlpack/core/data/check_categorical_param.hpp | 2 ++ src/mlpack/core/util/binding_details.hpp | 2 +- src/mlpack/core/util/forward.hpp | 5 ++++- src/mlpack/core/util/io.hpp | 1 + src/mlpack/core/util/params.hpp | 5 ++--- src/mlpack/core/util/params_impl.hpp | 1 + 6 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/mlpack/core/data/check_categorical_param.hpp b/src/mlpack/core/data/check_categorical_param.hpp index 988b978878..128ed9dbed 100644 --- a/src/mlpack/core/data/check_categorical_param.hpp +++ b/src/mlpack/core/data/check_categorical_param.hpp @@ -9,6 +9,8 @@ #ifndef MLPACK_CORE_DATA_CHECK_CATEGORICAL_PARAM_HPP #define MLPACK_CORE_DATA_CHECK_CATEGORICAL_PARAM_HPP +#include + namespace mlpack { namespace data { diff --git a/src/mlpack/core/util/binding_details.hpp b/src/mlpack/core/util/binding_details.hpp index 7e0b428d90..f5d0bc30f1 100644 --- a/src/mlpack/core/util/binding_details.hpp +++ b/src/mlpack/core/util/binding_details.hpp @@ -12,7 +12,7 @@ #ifndef MLPACK_CORE_UTIL_BINDING_DETAILS_HPP #define MLPACK_CORE_UTIL_BINDING_DETAILS_HPP -#include +#include namespace mlpack { namespace util { diff --git a/src/mlpack/core/util/forward.hpp b/src/mlpack/core/util/forward.hpp index edccc5c6e8..f25e3fb8f1 100644 --- a/src/mlpack/core/util/forward.hpp +++ b/src/mlpack/core/util/forward.hpp @@ -17,11 +17,14 @@ class IO; namespace util { -class Params; class Timers; } // namespace util +} +#include "params.hpp" + +namespace mlpack { namespace data { class IncrementPolicy; diff --git a/src/mlpack/core/util/io.hpp b/src/mlpack/core/util/io.hpp index a91359bb54..f15d6f067a 100644 --- a/src/mlpack/core/util/io.hpp +++ b/src/mlpack/core/util/io.hpp @@ -26,6 +26,7 @@ #include "param_data.hpp" #include "params.hpp" +#include "params_impl.hpp" #include #include diff --git a/src/mlpack/core/util/params.hpp b/src/mlpack/core/util/params.hpp index b4791d6937..e01b09b611 100644 --- a/src/mlpack/core/util/params.hpp +++ b/src/mlpack/core/util/params.hpp @@ -7,7 +7,7 @@ #ifndef MLPACK_CORE_UTIL_PARAMS_HPP #define MLPACK_CORE_UTIL_PARAMS_HPP -#include "forward.hpp" +//#include "forward.hpp" #include "param_data.hpp" #include "binding_details.hpp" #include @@ -146,7 +146,6 @@ class Params } // namespace util } // namespace mlpack -// Include implementation. -#include "params_impl.hpp" +// Implementation intentionally not included. #endif diff --git a/src/mlpack/core/util/params_impl.hpp b/src/mlpack/core/util/params_impl.hpp index 56a83f2af6..2914224c0c 100644 --- a/src/mlpack/core/util/params_impl.hpp +++ b/src/mlpack/core/util/params_impl.hpp @@ -9,6 +9,7 @@ #define MLPACK_CORE_UTIL_PARAMS_IMPL_HPP // Include definition, if needed. +#include "forward.hpp" #include "params.hpp" namespace mlpack { From 8e8b7fd70a81775da9fbc862fecd81f38b6d8923 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sat, 30 Jul 2022 08:14:58 -0400 Subject: [PATCH 38/39] Update standard to C++14. --- src/mlpack/bindings/R/mlpack/src/Makevars | 2 +- src/mlpack/bindings/R/mlpack/src/Makevars.win | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/bindings/R/mlpack/src/Makevars b/src/mlpack/bindings/R/mlpack/src/Makevars index bb6a88cc84..489fe04d78 100644 --- a/src/mlpack/bindings/R/mlpack/src/Makevars +++ b/src/mlpack/bindings/R/mlpack/src/Makevars @@ -1,3 +1,3 @@ PKG_CXXFLAGS = -DBOOST_MATH_PROMOTE_DOUBLE_POLICY=false -I. $(SHLIB_OPENMP_CXXFLAGS) PKG_LIBS = $(SHLIB_OPENMP_CXXFLAGS) $(LAPACK_LIBS) $(BLAS_LIBS) $(FLIBS) -CXX_STD = CXX11 +CXX_STD = CXX14 diff --git a/src/mlpack/bindings/R/mlpack/src/Makevars.win b/src/mlpack/bindings/R/mlpack/src/Makevars.win index 12f71348e6..cb4f589642 100644 --- a/src/mlpack/bindings/R/mlpack/src/Makevars.win +++ b/src/mlpack/bindings/R/mlpack/src/Makevars.win @@ -1,3 +1,3 @@ PKG_CXXFLAGS = -DBOOST_MATH_PROMOTE_DOUBLE_POLICY=false -I. $(SHLIB_OPENMP_CXXFLAGS) -ftrack-macro-expansion=0 -pipe --param ggc-min-expand=10 --param ggc-min-heapsize=8192 PKG_LIBS = $(SHLIB_OPENMP_CXXFLAGS) $(LAPACK_LIBS) $(BLAS_LIBS) $(FLIBS) -CXX_STD = CXX11 +CXX_STD = CXX14 From e5e8829ea3224a8a800f5984de011e7b82f34a07 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sun, 31 Jul 2022 07:39:45 -0400 Subject: [PATCH 39/39] Add a clarifying comment. --- src/mlpack/tests/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index dc9194b00a..8d01a61502 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -26,6 +26,8 @@ add_executable(mlpack_test ann/reward_clipping_test.cpp ann/rl_components_test.cpp + # These tests have not yet been adapted to the new inheritance-based ANN + # implementation. # ann/not_adapted/activation_functions_test.cpp # ann/not_adapted/ann_layer_test.cpp # ann/not_adapted/feedforward_network_test.cpp