From b624518249b4f19e5e8adf447ce0883f41e43d48 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Sun, 25 Feb 2018 22:06:12 +0530 Subject: [PATCH 01/79] Added base declaration --- .../nesterov_momentum_update.hpp | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp new file mode 100644 index 0000000000..eacad0f80f --- /dev/null +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -0,0 +1,27 @@ +/** + * @file nesterov_momentum_update.hpp + * @author Sourabh Varshney + * + * Nesterov Momentum Update for Stochastic Gradient Descent. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_SGD_NESTEROV_MOMENTUM_UPDATE_HPP +#define MLPACK_CORE_OPTIMIZERS_SGD_NESTEROV_MOMENTUM_UPDATE_HPP + +#include + +namespace mlpack { +namespace optimization { + +/* + */ + + +} // namespace optimization +} // namespace mlpack + +#endif From e51263eb8ef808b5f4c1b69a3ee4a4f9d576d7bb Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Sun, 25 Feb 2018 22:12:03 +0530 Subject: [PATCH 02/79] Made changes for adding nesterov momentum in sgd --- src/mlpack/core/optimizers/sgd/sgd.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mlpack/core/optimizers/sgd/sgd.hpp b/src/mlpack/core/optimizers/sgd/sgd.hpp index 023ee63950..f73e1f923a 100644 --- a/src/mlpack/core/optimizers/sgd/sgd.hpp +++ b/src/mlpack/core/optimizers/sgd/sgd.hpp @@ -3,6 +3,7 @@ * @author Ryan Curtin * @author Arun Reddy * @author Abhinav Moudgil + * @author Sourabh Varshney * * Stochastic Gradient Descent (SGD). * @@ -17,6 +18,7 @@ #include #include "update_policies/vanilla_update.hpp" #include "update_policies/momentum_update.hpp" +#include "update_policies/nesterov_momentum_update.hpp" #include "decay_policies/no_decay.hpp" namespace mlpack { @@ -202,6 +204,8 @@ using StandardSGD = SGD; using MomentumSGD = SGD; +using NesterovSGD = SGD; + } // namespace optimization } // namespace mlpack From e00a3fbf208cfb58a3df3d3d922d0e0c70a85081 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Mon, 26 Feb 2018 19:39:11 +0530 Subject: [PATCH 03/79] Added class declarations --- .../nesterov_momentum_update.hpp | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index eacad0f80f..8eeae6df27 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -20,6 +20,52 @@ namespace optimization { /* */ +class NesterovMomentumUpdate +{ + public: + /* + */ + NesterovMomentumUpdate(const double beta1 = 0.99 , + const double scheduleDecay = 4e-3) : + beta1(beta1), + scheduleDecay(scheduleDecay), + iteration(0) + { + // Nothing to do. + } + + /** + * The Initialize method is called by SGD Optimizer method before the start of + * the iteration update process. In the momentum update policy the velocity + * matrix is initialized to the zeros matrix with the same size as the + * gradient matrix (see mlpack::optimization::SGD::Optimizer ) + * + * @param rows Number of rows in the gradient matrix. + * @param cols Number of columns in the gradient matrix. + */ + void Initialize(const size_t rows, const size_t cols) + { + // Initialize am empty velocity matrix. + velocity = arma::zeros(rows, cols); + } + + /** + * Update step for SGD. The momentum term makes the convergence faster on the + * way as momentum term increases for dimensions pointing in the same and + * reduces updates for dimensions whose gradients change directions. + * + * @param iterate Parameters that minimize the function. + * @param stepSize Step size to be used for the given iteration. + * @param gradient The gradient matrix. + */ + void Update(arma::mat& iterate, + const double stepSize, + const arma::mat& gradient) + { + + } + +}; } // namespace optimization } // namespace mlpack From 409a2331940c1c6712724ba729072511dcfad055 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Mon, 26 Feb 2018 20:06:48 +0530 Subject: [PATCH 04/79] Add update formula --- .../nesterov_momentum_update.hpp | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 8eeae6df27..14964f720f 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -26,7 +26,7 @@ class NesterovMomentumUpdate /* */ NesterovMomentumUpdate(const double beta1 = 0.99 , - const double scheduleDecay = 4e-3) : + const double scheduleDecay = 4e-3) : beta1(beta1), scheduleDecay(scheduleDecay), iteration(0) @@ -62,9 +62,34 @@ class NesterovMomentumUpdate const double stepSize, const arma::mat& gradient) { - + double beta1T = beta1 * (1 - (0.5 * + std::pow(0.96, (iteration - 1) * scheduleDecay))); + + double beta1T1 = beta1 * (1 - (0.5 * + std::pow(0.96, iteration * scheduleDecay))); + + iterate = iterate + (beta1T * beta1T1 * velocity) - ((1 + beta1T1) * stepSize * gradient); + + velocity = beta1T * velocity - stepSize * gradient; } + //! Get the smoothing parameter. + double Beta1() const { return beta1; } + //! Modify the smoothing parameter. + double& Beta1() { return beta1; } + + //! Get the decay parameter for decay coefficients + double ScheduleDecay() const { return scheduleDecay; } + //! Modify the decay parameter for decay coefficients + double& ScheduleDecay() { return scheduleDecay; } + + private: + // The smoothing parameter. + double beta1; + + // The velocity matrix. + arma::mat velocity; + }; } // namespace optimization From 0e0cdbc768a069574568de2e9c771261778a9a51 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Mon, 26 Feb 2018 22:08:23 +0530 Subject: [PATCH 05/79] Add comments and reference paper --- .../nesterov_momentum_update.hpp | 37 +++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 14964f720f..918bb2c893 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -17,14 +17,37 @@ namespace mlpack { namespace optimization { -/* +/** + * Nesterov Momentum update policy for Stochastic Gradient Descent (SGD). + * + * Learning with SGD can be slow. Applying Standard momentum can accelerate + * the rate of convergence. Nesterov Momentum application can accelerate the + * rate of convergence to O(1/k^2). + * + * @code + * @techreport{Nesterov1983, + * title = {A Method Of Solving A Convex Programming Problem With + * Convergence Rate O(1/K^2)}, + * author = {Yuri Nesterov}, + * institution = {Soviet Math. Dokl.}, + * volume = {27}, + * year = {1983}, + * url = {http://www.cis.pku.edu.cn/faculty/vision/zlin/1983-A%20 + Method%20of%20Solving%20a%20Convex%20Programming%20Problem + %20with%20Convergence%20Rate%20O(k%5E(-2))_Nesterov.pdf} + * } + * @endcode */ class NesterovMomentumUpdate { public: - /* - */ + /** + * Construct the Nesterov Momentum update policy with the given parameters. + * + * @param beta1 The second moment coefficient. + * @param scheduleDecay The decay parameter for decay coefficients + */ NesterovMomentumUpdate(const double beta1 = 0.99 , const double scheduleDecay = 4e-3) : beta1(beta1), @@ -70,12 +93,12 @@ class NesterovMomentumUpdate iterate = iterate + (beta1T * beta1T1 * velocity) - ((1 + beta1T1) * stepSize * gradient); - velocity = beta1T * velocity - stepSize * gradient; + velocity = beta1T * velocity - stepSize * gradient;pd } - //! Get the smoothing parameter. + //! Get the second moment coefficient. double Beta1() const { return beta1; } - //! Modify the smoothing parameter. + //! Modify the second moment coefficient. double& Beta1() { return beta1; } //! Get the decay parameter for decay coefficients @@ -84,7 +107,7 @@ class NesterovMomentumUpdate double& ScheduleDecay() { return scheduleDecay; } private: - // The smoothing parameter. + // The second moment coefficient. double beta1; // The velocity matrix. From ebb37286dd0d8f23ac60860fa6c9493a152c58d2 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Mon, 26 Feb 2018 22:11:48 +0530 Subject: [PATCH 06/79] Add method in Cmakelist --- src/mlpack/core/optimizers/sgd/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/core/optimizers/sgd/CMakeLists.txt b/src/mlpack/core/optimizers/sgd/CMakeLists.txt index 343d024d90..b4c266316c 100644 --- a/src/mlpack/core/optimizers/sgd/CMakeLists.txt +++ b/src/mlpack/core/optimizers/sgd/CMakeLists.txt @@ -2,6 +2,7 @@ set(SOURCES decay_policies/no_decay.hpp update_policies/gradient_clipping.hpp update_policies/momentum_update.hpp + update_policies/nesterov_momentum_update.hpp update_policies/vanilla_update.hpp sgd.hpp sgd_impl.hpp From e42f9a84ccc6e12dda748d70b57781f4316ef0be Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Tue, 27 Feb 2018 11:27:56 +0530 Subject: [PATCH 07/79] Removed typo mistakes --- src/mlpack/core/optimizers/sgd/sgd.hpp | 2 +- .../sgd/update_policies/nesterov_momentum_update.hpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/sgd.hpp b/src/mlpack/core/optimizers/sgd/sgd.hpp index f73e1f923a..523539ec73 100644 --- a/src/mlpack/core/optimizers/sgd/sgd.hpp +++ b/src/mlpack/core/optimizers/sgd/sgd.hpp @@ -204,7 +204,7 @@ using StandardSGD = SGD; using MomentumSGD = SGD; -using NesterovSGD = SGD; +using NesterovMomentumSGD = SGD; } // namespace optimization } // namespace mlpack diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 918bb2c893..3e0bc6899d 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -49,7 +49,7 @@ class NesterovMomentumUpdate * @param scheduleDecay The decay parameter for decay coefficients */ NesterovMomentumUpdate(const double beta1 = 0.99 , - const double scheduleDecay = 4e-3) : + const double scheduleDecay = 4e-3) : beta1(beta1), scheduleDecay(scheduleDecay), iteration(0) @@ -93,7 +93,7 @@ class NesterovMomentumUpdate iterate = iterate + (beta1T * beta1T1 * velocity) - ((1 + beta1T1) * stepSize * gradient); - velocity = beta1T * velocity - stepSize * gradient;pd + velocity = beta1T * velocity - stepSize * gradient; } //! Get the second moment coefficient. From e50eb991589109d3ae68561fdac01034007bbf5e Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Tue, 27 Feb 2018 11:29:19 +0530 Subject: [PATCH 08/79] Add tests for nesterov momentum sgd --- .../tests/nesterov_momentum_sgd_test.cpp | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 src/mlpack/tests/nesterov_momentum_sgd_test.cpp diff --git a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp new file mode 100644 index 0000000000..ee63b260be --- /dev/null +++ b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp @@ -0,0 +1,81 @@ +/** + * @file nesterov_momentum_sgd_test.cpp + * @author Sourabh Varshney + * + * Test file for NesterovMomentumSGD (Stochastic gradient descent with + * nesterov momentum updates). + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include +#include +#include +#include +#include +#include + +#include +#include "test_tools.hpp" + +using namespace std; +using namespace arma; +using namespace mlpack; +using namespace mlpack::optimization; +using namespace mlpack::optimization::test; + +BOOST_AUTO_TEST_SUITE(NesterovMomentumSGDTest); + +BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) +{ + SGDTestFunction f; + NesterovMomentumUpdate nesterovMomentumUpdate(0.99, 4e-3); + NesterovMomentumSGD s(0.0003, 1, 2500000, 1e-9, true, + nesterovMomentumUpdate); + + arma::mat coordinates = f.GetInitialPoint(); + double result = s.Optimize(f, coordinates); + + BOOST_REQUIRE_CLOSE(result, -1.0, 0.15); + BOOST_REQUIRE_SMALL(coordinates[0], 1e-3); + BOOST_REQUIRE_SMALL(coordinates[1], 1e-7); + BOOST_REQUIRE_SMALL(coordinates[2], 1e-7); + + // Compare with SGD with vanilla update. + SGDTestFunction f1; + StandardSGD s1(0.0003, 1, 2500000, 1e-9, true); + + arma::mat coordinates1 = f.GetInitialPoint(); + double result1 = s1.Optimize(f1, coordinates1); + + // Result doesn't converge in 2500000 iterations. + BOOST_REQUIRE_GT(result1 + 1.0, 0.05); + BOOST_REQUIRE_GE(coordinates1[0], 1e-3); + BOOST_REQUIRE_SMALL(coordinates1[1], 1e-7); + BOOST_REQUIRE_SMALL(coordinates1[2], 1e-7); + + BOOST_REQUIRE_LE(result, result1); +} + +BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) +{ + // Loop over several variants. + for (size_t i = 10; i < 50; i += 5) + { + // Create the generalized Rosenbrock function. + GeneralizedRosenbrockFunction f(i); + NesterovMomentumUpdate nesterovMomentumUpdate(0.88, 4e-3); + NesterovMomentumSGD s(0.0008, 1, 0, 1e-15, true, nesterovMomentumUpdate); + + arma::mat coordinates = f.GetInitialPoint(); + double result = s.Optimize(f, coordinates); + + BOOST_REQUIRE_SMALL(result, 1e-4); + for (size_t j = 0; j < i; ++j) + BOOST_REQUIRE_CLOSE(coordinates[j], (double) 1.0, 1e-3); + } +} + +BOOST_AUTO_TEST_SUITE_END(); From e7dea3a553448518427cae36f6580610b327465d Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Tue, 27 Feb 2018 12:27:40 +0530 Subject: [PATCH 09/79] Fix style checks and declared remaining variables --- .../update_policies/nesterov_momentum_update.hpp | 16 +++++++++++----- src/mlpack/tests/nesterov_momentum_sgd_test.cpp | 2 +- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 3e0bc6899d..45ef0026e2 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -49,7 +49,7 @@ class NesterovMomentumUpdate * @param scheduleDecay The decay parameter for decay coefficients */ NesterovMomentumUpdate(const double beta1 = 0.99 , - const double scheduleDecay = 4e-3) : + const double scheduleDecay = 4e-3) : beta1(beta1), scheduleDecay(scheduleDecay), iteration(0) @@ -85,15 +85,16 @@ class NesterovMomentumUpdate const double stepSize, const arma::mat& gradient) { - double beta1T = beta1 * (1 - (0.5 * + double beta1T = beta1 * (1 - (0.5 * std::pow(0.96, (iteration - 1) * scheduleDecay))); double beta1T1 = beta1 * (1 - (0.5 * std::pow(0.96, iteration * scheduleDecay))); - iterate = iterate + (beta1T * beta1T1 * velocity) - ((1 + beta1T1) * stepSize * gradient); - - velocity = beta1T * velocity - stepSize * gradient; + iterate = iterate + (beta1T * beta1T1 * velocity) - ((1 + beta1T1) + * stepSize * gradient); + + velocity = beta1T * velocity - stepSize * gradient; } //! Get the second moment coefficient. @@ -113,6 +114,11 @@ class NesterovMomentumUpdate // The velocity matrix. arma::mat velocity; + // The decay parameter for decay coefficients. + double scheduleDecay; + + // The number of iterations. + double iteration; }; } // namespace optimization diff --git a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp index ee63b260be..37e3ed3b74 100644 --- a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp +++ b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp @@ -33,7 +33,7 @@ BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) SGDTestFunction f; NesterovMomentumUpdate nesterovMomentumUpdate(0.99, 4e-3); NesterovMomentumSGD s(0.0003, 1, 2500000, 1e-9, true, - nesterovMomentumUpdate); + nesterovMomentumUpdate); arma::mat coordinates = f.GetInitialPoint(); double result = s.Optimize(f, coordinates); From 7cf5718b04d00706e781c0e6d26b675b0bf28a85 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Tue, 27 Feb 2018 12:40:02 +0530 Subject: [PATCH 10/79] Mention test for nesterov sgd in CMakelists --- src/mlpack/tests/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 69910edfa8..11466a9098 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -74,6 +74,7 @@ add_executable(mlpack_test momentum_sgd_test.cpp nbc_test.cpp nca_test.cpp + nesterov_momentum_sgd_test.cpp nmf_test.cpp nystroem_method_test.cpp octree_test.cpp From d27c95f8da2521ffb5d742598ab21b0e33c7205f Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Wed, 28 Feb 2018 10:18:08 +0530 Subject: [PATCH 11/79] Solved pedantic style issues --- .../sgd/update_policies/nesterov_momentum_update.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 45ef0026e2..f571a42b64 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -27,14 +27,11 @@ namespace optimization { * @code * @techreport{Nesterov1983, * title = {A Method Of Solving A Convex Programming Problem With - * Convergence Rate O(1/K^2)}, + * Convergence Rate O(1/K^2)}, * author = {Yuri Nesterov}, * institution = {Soviet Math. Dokl.}, * volume = {27}, * year = {1983}, - * url = {http://www.cis.pku.edu.cn/faculty/vision/zlin/1983-A%20 - Method%20of%20Solving%20a%20Convex%20Programming%20Problem - %20with%20Convergence%20Rate%20O(k%5E(-2))_Nesterov.pdf} * } * @endcode */ From c0c0f2befb1701ec157d071fd09ca635f378329f Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Thu, 1 Mar 2018 23:04:55 +0530 Subject: [PATCH 12/79] Updated nesterov_momentum formula --- .../nesterov_momentum_update.hpp | 35 +++---------------- 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index f571a42b64..96f01b5824 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -42,14 +42,8 @@ class NesterovMomentumUpdate /** * Construct the Nesterov Momentum update policy with the given parameters. * - * @param beta1 The second moment coefficient. - * @param scheduleDecay The decay parameter for decay coefficients */ - NesterovMomentumUpdate(const double beta1 = 0.99 , - const double scheduleDecay = 4e-3) : - beta1(beta1), - scheduleDecay(scheduleDecay), - iteration(0) + NesterovMomentumUpdate() : iteration(0) { // Nothing to do. } @@ -82,38 +76,19 @@ class NesterovMomentumUpdate const double stepSize, const arma::mat& gradient) { - double beta1T = beta1 * (1 - (0.5 * - std::pow(0.96, (iteration - 1) * scheduleDecay))); + iteration++; - double beta1T1 = beta1 * (1 - (0.5 * - std::pow(0.96, iteration * scheduleDecay))); + double momentum = 1 - (3 / (iteration + 5)); - iterate = iterate + (beta1T * beta1T1 * velocity) - ((1 + beta1T1) - * stepSize * gradient); + velocity = momentum * velocity - stepSize * gradient; - velocity = beta1T * velocity - stepSize * gradient; + iterate += velocity; } - //! Get the second moment coefficient. - double Beta1() const { return beta1; } - //! Modify the second moment coefficient. - double& Beta1() { return beta1; } - - //! Get the decay parameter for decay coefficients - double ScheduleDecay() const { return scheduleDecay; } - //! Modify the decay parameter for decay coefficients - double& ScheduleDecay() { return scheduleDecay; } - private: - // The second moment coefficient. - double beta1; - // The velocity matrix. arma::mat velocity; - // The decay parameter for decay coefficients. - double scheduleDecay; - // The number of iterations. double iteration; }; From e3641f3f0fb19b859bc227c700b4d553e7231205 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Thu, 1 Mar 2018 23:05:50 +0530 Subject: [PATCH 13/79] Updated tests for nesterov momentum --- src/mlpack/tests/nesterov_momentum_sgd_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp index 37e3ed3b74..39c1e02119 100644 --- a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp +++ b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp @@ -31,7 +31,7 @@ BOOST_AUTO_TEST_SUITE(NesterovMomentumSGDTest); BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) { SGDTestFunction f; - NesterovMomentumUpdate nesterovMomentumUpdate(0.99, 4e-3); + NesterovMomentumUpdate nesterovMomentumUpdate(); NesterovMomentumSGD s(0.0003, 1, 2500000, 1e-9, true, nesterovMomentumUpdate); @@ -66,7 +66,7 @@ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) { // Create the generalized Rosenbrock function. GeneralizedRosenbrockFunction f(i); - NesterovMomentumUpdate nesterovMomentumUpdate(0.88, 4e-3); + NesterovMomentumUpdate nesterovMomentumUpdate(); NesterovMomentumSGD s(0.0008, 1, 0, 1e-15, true, nesterovMomentumUpdate); arma::mat coordinates = f.GetInitialPoint(); From bb24b37ea3ef6ab7abd23a59f3bcccd6527300d8 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Thu, 1 Mar 2018 23:34:50 +0530 Subject: [PATCH 14/79] Corrected declaration --- src/mlpack/tests/nesterov_momentum_sgd_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp index 39c1e02119..18ab96ae0b 100644 --- a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp +++ b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp @@ -31,7 +31,7 @@ BOOST_AUTO_TEST_SUITE(NesterovMomentumSGDTest); BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) { SGDTestFunction f; - NesterovMomentumUpdate nesterovMomentumUpdate(); + NesterovMomentumUpdate nesterovMomentumUpdate; NesterovMomentumSGD s(0.0003, 1, 2500000, 1e-9, true, nesterovMomentumUpdate); @@ -66,7 +66,7 @@ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) { // Create the generalized Rosenbrock function. GeneralizedRosenbrockFunction f(i); - NesterovMomentumUpdate nesterovMomentumUpdate(); + NesterovMomentumUpdate nesterovMomentumUpdate; NesterovMomentumSGD s(0.0008, 1, 0, 1e-15, true, nesterovMomentumUpdate); arma::mat coordinates = f.GetInitialPoint(); From a2ffa3530813d41b6c24f8d5bd8c3ce0295944b9 Mon Sep 17 00:00:00 2001 From: Moksh Jain Date: Thu, 8 Mar 2018 14:41:56 +0530 Subject: [PATCH 15/79] added adamirror optmizer --- .../core/optimizers/adam/CMakeLists.txt | 1 + src/mlpack/core/optimizers/adam/adam.hpp | 3 + .../core/optimizers/adam/adamirror_update.hpp | 144 ++++++++++++++++++ src/mlpack/tests/adam_test.cpp | 76 +++++++++ 4 files changed, 224 insertions(+) create mode 100644 src/mlpack/core/optimizers/adam/adamirror_update.hpp diff --git a/src/mlpack/core/optimizers/adam/CMakeLists.txt b/src/mlpack/core/optimizers/adam/CMakeLists.txt index 6340ba68ba..908fe06db9 100644 --- a/src/mlpack/core/optimizers/adam/CMakeLists.txt +++ b/src/mlpack/core/optimizers/adam/CMakeLists.txt @@ -6,6 +6,7 @@ set(SOURCES amsgrad_update.hpp nadam_update.hpp nadamax_update.hpp + adamirror_update.hpp ) set(DIR_SRCS) diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp index 0b3b2a5a14..f0c209cf01 100644 --- a/src/mlpack/core/optimizers/adam/adam.hpp +++ b/src/mlpack/core/optimizers/adam/adam.hpp @@ -30,6 +30,7 @@ #include "amsgrad_update.hpp" #include "nadam_update.hpp" #include "nadamax_update.hpp" +#include "adamirror_update.hpp" namespace mlpack { namespace optimization { @@ -186,6 +187,8 @@ using Nadam = AdamType; using NadaMax = AdamType; +using Adamirror = AdamType; + } // namespace optimization } // namespace mlpack diff --git a/src/mlpack/core/optimizers/adam/adamirror_update.hpp b/src/mlpack/core/optimizers/adam/adamirror_update.hpp new file mode 100644 index 0000000000..1e934c3b75 --- /dev/null +++ b/src/mlpack/core/optimizers/adam/adamirror_update.hpp @@ -0,0 +1,144 @@ +/** + * @file adamirror_update.hpp + * @author Moksh Jain + * + * Adamirror optimizer. Optimistic Adam is an an algorithm which uses + * Optimistic Mirror Descent with the Adam optimizer. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_ADAM_ADAMIRROR_UPDATE_HPP +#define MLPACK_CORE_OPTIMIZERS_ADAM_ADAMIRROR_UPDATE_HPP + +#include + +namespace mlpack { +namespace optimization { + +/** + * Adamirror is an optimizer which uses Optmistic Mirror Descent with + * the Adam Optimizer. + * + * For more information, see the following. + * + * @code + * @article{ + * author = {Constantinos Daskalakis, Andrew Ilyas, Vasilis Syrgkanis, + * Haoyang Zeng}, + * title = {Training GANs with Optimism}, + * year = {2017}, + * url = {https://openreview.net/forum?id=SJJySbbAZ} + * } + * @endcode + */ +class AdamirrorUpdate +{ + public: + /** + * Construct the Adamirror update policy with the given parameters. + * + * @param epsilon The epsilon value used to initialise the squared gradient + * parameter. + * @param beta1 The smoothing parameter. + * @param beta2 The second moment coefficient. + */ + AdamirrorUpdate(const double epsilon = 1e-8, + const double beta1 = 0.9, + const double beta2 = 0.999) : + epsilon(epsilon), + beta1(beta1), + beta2(beta2), + iteration(0) + { + // Nothing to do. + } + + /** + * The Initialize method is called by SGD Optimizer method before the start of + * the iteration update process. + * + * @param rows Number of rows in the gradient matrix. + * @param cols Number of columns in the gradient matrix. + */ + void Initialize(const size_t rows, const size_t cols) + { + m = arma::zeros(rows, cols); + v = arma::zeros(rows, cols); + g = arma::zeros(rows, cols); + } + + /** + * Update step for Adamirror. + * + * @param iterate Parameters that minimize the function. + * @param stepSize Step size to be used for the given iteration. + * @param gradient The gradient matrix. + */ + void Update(arma::mat& iterate, + const double stepSize, + const arma::mat& gradient) + { + // Increment the iteration counter variable. + ++iteration; + + // And update the iterate. + m *= beta1; + m += (1 - beta1) * gradient; + + v *= beta2; + v += (1 - beta2) * arma::square(gradient); + + arma::mat mCorrected = m / (1.0 - std::pow(beta1, iteration)); + arma::mat vCorrected = v / (1.0 - std::pow(beta2, iteration)); + + arma::mat update = mCorrected / (arma::sqrt(vCorrected) + epsilon); + + iterate -= (2 * stepSize * update - stepSize * g); + + g = update; + } + + //! Get the value used to initialise the squared gradient parameter. + double Epsilon() const { return epsilon; } + //! Modify the value used to initialise the squared gradient parameter. + double& Epsilon() { return epsilon; } + + //! Get the smoothing parameter. + double Beta1() const { return beta1; } + //! Modify the smoothing parameter. + double& Beta1() { return beta1; } + + //! Get the second moment coefficient. + double Beta2() const { return beta2; } + //! Modify the second moment coefficient. + double& Beta2() { return beta2; } + + private: + // The epsilon value used to initialise the squared gradient parameter. + double epsilon; + + // The smoothing parameter. + double beta1; + + // The second moment coefficient. + double beta2; + + // The exponential moving average of gradient values. + arma::mat m; + + // The exponential moving average of squared gradient values. + arma::mat v; + // The previous update + arma::mat g; + + // The number of iterations. + double iteration; +}; + +} // namespace optimization +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/adam_test.cpp b/src/mlpack/tests/adam_test.cpp index 0d581cffdd..588e94e8d5 100644 --- a/src/mlpack/tests/adam_test.cpp +++ b/src/mlpack/tests/adam_test.cpp @@ -507,4 +507,80 @@ BOOST_AUTO_TEST_CASE(NadaMaxLogisticRegressionTest) BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. } +/** + * Tests the Adamirror optimizer using a simple test function. + */ +BOOST_AUTO_TEST_CASE(SimpleAdamirrorTestFunction) +{ + SGDTestFunction f; + Adamirror optimizer(1e-3, 1, 0.9, 0.99, 1e-8); + + arma::mat coordinates = f.GetInitialPoint(); + optimizer.Optimize(f, coordinates); + + BOOST_REQUIRE_SMALL(coordinates[0], 0.1); + BOOST_REQUIRE_SMALL(coordinates[1], 0.1); + BOOST_REQUIRE_SMALL(coordinates[2], 0.1); +} + +/** + * Run Adamirror on logistic regression and make sure the results are acceptable. + */ +BOOST_AUTO_TEST_CASE(AdamirrorLogisticRegressionTest) +{ + // Generate a two-Gaussian dataset. + GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), + arma::eye(3, 3)); + GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), + arma::eye(3, 3)); + + arma::mat data(3, 1000); + arma::Row responses(1000); + for (size_t i = 0; i < 500; ++i) + { + data.col(i) = g1.Random(); + responses[i] = 0; + } + for (size_t i = 500; i < 1000; ++i) + { + data.col(i) = g2.Random(); + responses[i] = 1; + } + + // Shuffle the dataset. + arma::uvec indices = arma::shuffle(arma::linspace(0, + data.n_cols - 1, data.n_cols)); + arma::mat shuffledData(3, 1000); + arma::Row shuffledResponses(1000); + for (size_t i = 0; i < data.n_cols; ++i) + { + shuffledData.col(i) = data.col(indices[i]); + shuffledResponses[i] = responses[indices[i]]; + } + + // Create a test set. + arma::mat testData(3, 1000); + arma::Row testResponses(1000); + for (size_t i = 0; i < 500; ++i) + { + testData.col(i) = g1.Random(); + testResponses[i] = 0; + } + for (size_t i = 500; i < 1000; ++i) + { + testData.col(i) = g2.Random(); + testResponses[i] = 1; + } + + Adamirror adamirror; + LogisticRegression<> lr(shuffledData, shuffledResponses, adamirror, 0.5); + + // Ensure that the error is close to zero. + const double acc = lr.ComputeAccuracy(data, responses); + BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. + + const double testAcc = lr.ComputeAccuracy(testData, testResponses); + BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. +} + BOOST_AUTO_TEST_SUITE_END(); From 80de5c8213907a461eb67fb937170042d079ed37 Mon Sep 17 00:00:00 2001 From: Moksh Jain Date: Fri, 9 Mar 2018 01:39:48 +0530 Subject: [PATCH 16/79] fix adamirror test case --- src/mlpack/tests/adam_test.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mlpack/tests/adam_test.cpp b/src/mlpack/tests/adam_test.cpp index 588e94e8d5..279c8e5ffd 100644 --- a/src/mlpack/tests/adam_test.cpp +++ b/src/mlpack/tests/adam_test.cpp @@ -12,6 +12,7 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ + #include #include @@ -513,7 +514,7 @@ BOOST_AUTO_TEST_CASE(NadaMaxLogisticRegressionTest) BOOST_AUTO_TEST_CASE(SimpleAdamirrorTestFunction) { SGDTestFunction f; - Adamirror optimizer(1e-3, 1, 0.9, 0.99, 1e-8); + Adamirror optimizer(1e-2, 1, 0.9, 0.99, 1e-8); arma::mat coordinates = f.GetInitialPoint(); optimizer.Optimize(f, coordinates); From 4c818eac5f3ca2a3baaaecf88f0bc5d515c3ac52 Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Tue, 6 Mar 2018 13:19:49 +0530 Subject: [PATCH 17/79] Early stopping parameter --- .../decision_tree/all_categorical_split.hpp | 3 +- .../all_categorical_split_impl.hpp | 5 +- .../best_binary_numeric_split.hpp | 3 +- .../best_binary_numeric_split_impl.hpp | 5 +- .../methods/decision_tree/decision_tree.hpp | 32 +++++++++-- .../decision_tree/decision_tree_impl.hpp | 55 ++++++++++++------- .../decision_tree/decision_tree_main.cpp | 12 ++-- 7 files changed, 78 insertions(+), 37 deletions(-) diff --git a/src/mlpack/methods/decision_tree/all_categorical_split.hpp b/src/mlpack/methods/decision_tree/all_categorical_split.hpp index b92702ebc9..4116806fac 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split.hpp @@ -62,7 +62,8 @@ class AllCategoricalSplit const WeightVecType& weights, const size_t minimumLeafSize, arma::Col& classProbabilities, - AuxiliarySplitInfo& aux); + AuxiliarySplitInfo& aux, + const double minimumGainSplit); /** * Return the number of children in the split. diff --git a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp index b15c7381f4..a00a7b5957 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp @@ -26,7 +26,8 @@ double AllCategoricalSplit::SplitIfBetter( const WeightVecType& weights, const size_t minimumLeafSize, arma::Col& classProbabilities, - AuxiliarySplitInfo& /* aux */) + AuxiliarySplitInfo& /* aux */, + const double minimumGainSplit) { // Count the number of elements in each potential child. const double epsilon = 1e-7; // Tolerance for floating-point errors. @@ -96,7 +97,7 @@ double AllCategoricalSplit::SplitIfBetter( overallGain += childPct * childGain; } - if (overallGain > bestGain + epsilon) + if (overallGain > bestGain + minimumGainSplit + epsilon) { // This is better, so set up the class probabilities vector and return. classProbabilities.set_size(1); diff --git a/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp b/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp index 4b0f039fad..644145ec96 100644 --- a/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp +++ b/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp @@ -59,7 +59,8 @@ class BestBinaryNumericSplit const WeightVecType& weights, const size_t minimumLeafSize, arma::Col& classProbabilities, - AuxiliarySplitInfo& aux); + AuxiliarySplitInfo& aux, + const double minimumGainSplit); /** * Returns 2, since the binary split always has two children. diff --git a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp b/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp index 82413e796f..13e99b5ea9 100644 --- a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp @@ -25,7 +25,8 @@ double BestBinaryNumericSplit::SplitIfBetter( const WeightVecType& weights, const size_t minimumLeafSize, arma::Col& classProbabilities, - AuxiliarySplitInfo& /* aux */) + AuxiliarySplitInfo& /* aux */, + const double minimumGainSplit) { // First sanity check: if we don't have enough points, we can't split. if (data.n_elem < (minimumLeafSize * 2)) @@ -104,7 +105,7 @@ double BestBinaryNumericSplit::SplitIfBetter( data[sortedIndices[index]]) / 2.0; return gain; } - else if (gain > bestFoundGain) + else if (gain > bestFoundGain + minimumGainSplit) { // We still have a better split. bestFoundGain = gain; diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp index c52ee9fe2e..7a4cacd0b9 100644 --- a/src/mlpack/methods/decision_tree/decision_tree.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree.hpp @@ -61,13 +61,15 @@ class DecisionTree : * @param labels Labels for each training point. * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in each leaf node. + * @param minimumGainSplit Minimum Gain for the node to split. */ template DecisionTree(MatType&& data, const data::DatasetInfo& datasetInfo, LabelsType&& labels, const size_t numClasses, - const size_t minimumLeafSize = 10); + const size_t minimumLeafSize = 10, + const double minimumGainSplit = 1e-7); /** * Construct the decision tree on the given data and labels, assuming that the @@ -79,12 +81,14 @@ class DecisionTree : * @param labels Labels for each training point. * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in each leaf node. + * @param minimumGainSplit Minimum Gain for the node to split. */ template DecisionTree(MatType&& data, LabelsType&& labels, const size_t numClasses, - const size_t minimumLeafSize = 10); + const size_t minimumLeafSize = 10, + const double minimumGainSplit = 1e-7); /** * Construct the decision tree on the given data and labels with weights, @@ -98,6 +102,7 @@ class DecisionTree : * @param numClasses Number of classes in the dataset. * @param weights The weight list of given label. * @param minimumLeafSize Minimum number of points in each leaf node. + * @param minimumGainSplit Minimum Gain for the node to split. */ template DecisionTree(MatType&& data, @@ -106,6 +111,7 @@ class DecisionTree : const size_t numClasses, WeightsType&& weights, const size_t minimumLeafSize = 10, + const double minimumGainSplit = 1e-7, const std::enable_if_t::type>::value>* = 0); @@ -121,6 +127,7 @@ class DecisionTree : * @param numClasses Number of classes in the dataset. * @param weights The Weight list of given labels. * @param minimumLeafSize Minimum number of points in each leaf node. + * @param minimumGainSplit Minimum Gain for the node to split. */ template DecisionTree(MatType&& data, @@ -128,6 +135,7 @@ class DecisionTree : const size_t numClasses, WeightsType&& weights, const size_t minimumLeafSize = 10, + const double minimumGainSplit = 1e-7, const std::enable_if_t::type>::value>* = 0); @@ -188,13 +196,15 @@ class DecisionTree : * @param numClasses Number of classes in the dataset. * @param weights Weights of all the labels * @param minimumLeafSize Minimum number of points in each leaf node. + * @param minimumGainSplit Minimum Gain for the node to split. */ template void Train(MatType&& data, const data::DatasetInfo& datasetInfo, LabelsType&& labels, const size_t numClasses, - const size_t minimumLeafSize = 10); + const size_t minimumLeafSize = 10, + const double minimumGainSplit = 1e-7); /** * Train the decision tree on the given data, assuming that all dimensions are @@ -207,12 +217,14 @@ class DecisionTree : * @param numClasses Number of classes in the dataset. * @param weights Weights of all the labels * @param minimumLeafSize Minimum number of points in each leaf node. + * @param minimumGainSplit Minimum Gain for the node to split. */ template void Train(MatType&& data, LabelsType&& labels, const size_t numClasses, - const size_t minimumLeafSize = 10); + const size_t minimumLeafSize = 10, + const double minimumGainSplit = 1e-7); /** * Train the decision tree on the given weighted data. This will overwrite @@ -227,6 +239,7 @@ class DecisionTree : * @param numClasses Number of classes in the dataset. * @param weights Weights of all the labels * @param minimumLeafSize Minimum number of points in each leaf node. + * @param minimumGainSplit Minimum Gain for the node to split. */ template void Train(MatType&& data, @@ -235,6 +248,7 @@ class DecisionTree : const size_t numClasses, WeightsType&& weights, const size_t minimumLeafSize = 10, + const double minimumGainSplit = 1e-7, const std::enable_if_t::type>::value>* = 0); @@ -249,6 +263,7 @@ class DecisionTree : * @param numClasses Number of classes in the dataset. * @param weights Weights of all the labels * @param minimumLeafSize Minimum number of points in each leaf node. + * @param minimumGainSplit Minimum Gain for the node to split. */ template void Train(MatType&& data, @@ -256,6 +271,7 @@ class DecisionTree : const size_t numClasses, WeightsType&& weights, const size_t minimumLeafSize = 10, + const double minimumGainSplit = 1e-7, const std::enable_if_t::type>::value>* = 0); @@ -383,6 +399,7 @@ class DecisionTree : * @param labels Labels for each training point. * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in each leaf node. + * @param minimumGainSplit Minimum Gain for the node to split. */ template void Train(MatType& data, @@ -392,7 +409,8 @@ class DecisionTree : arma::Row& labels, const size_t numClasses, arma::rowvec& weights, - const size_t minimumLeafSize = 10); + const size_t minimumLeafSize = 10, + const double minimumGainSplit = 1e-7); /** * Corresponding to the public Train() method, this method is designed for @@ -406,6 +424,7 @@ class DecisionTree : * @param labels Labels for each training point. * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in each leaf node. + * @param minimumGainSplit Minimum Gain for the node to split. */ template void Train(MatType& data, @@ -414,7 +433,8 @@ class DecisionTree : arma::Row& labels, const size_t numClasses, arma::rowvec& weights, - const size_t minimumLeafSize = 10); + const size_t minimumLeafSize = 10, + const double minimumGainSplit = 1e-7); }; /** diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index a855e02b34..10b2a45cb1 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -32,7 +32,8 @@ DecisionTree::type; using TrueLabelsType = typename std::decay::type; @@ -44,7 +45,7 @@ DecisionTree(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, numClasses, - weights, minimumLeafSize); + weights, minimumLeafSize, minimumGainSplit); } //! Construct and train. @@ -63,7 +64,8 @@ DecisionTree::DecisionTree(MatType&& data, LabelsType&& labels, const size_t numClasses, - const size_t minimumLeafSize) + const size_t minimumLeafSize, + const double minimumGainSplit) { using TrueMatType = typename std::decay::type; using TrueLabelsType = typename std::decay::type; @@ -75,7 +77,7 @@ DecisionTree(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, weights, - minimumLeafSize); + minimumLeafSize, minimumGainSplit); } //! Construct and train with weights. @@ -97,6 +99,7 @@ DecisionTree(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, numClasses, - tmpWeights, minimumLeafSize); + tmpWeights, minimumLeafSize, minimumGainSplit); } //! Construct and train with weights. @@ -134,6 +137,7 @@ DecisionTree(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, tmpWeights, - minimumLeafSize); + minimumLeafSize, minimumGainSplit); } //! Construct, don't train. @@ -345,7 +349,8 @@ void DecisionTree(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, numClasses, - weights, minimumLeafSize); + weights, minimumLeafSize, minimumGainSplit); } //! Train on the given data, assuming all dimensions are numeric. @@ -386,7 +391,8 @@ void DecisionTree::Train(MatType&& data, LabelsType&& labels, const size_t numClasses, - const size_t minimumLeafSize) + const size_t minimumLeafSize, + const double minimumGainSplit) { // Sanity check on data. if (data.n_cols != labels.n_elem) @@ -430,6 +436,7 @@ void DecisionTree::type>::value>*) @@ -455,7 +462,7 @@ void DecisionTree(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, numClasses, - tmpWeights, minimumLeafSize); + tmpWeights, minimumLeafSize, minimumGainSplit); } //! Train on the given weighted data. @@ -476,6 +483,7 @@ void DecisionTree::type>::value>*) @@ -501,7 +509,7 @@ void DecisionTree(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, tmpWeights, - minimumLeafSize); + minimumLeafSize, minimumGainSplit); } //! Train on the given data. @@ -524,7 +532,8 @@ void DecisionTree& labels, const size_t numClasses, arma::rowvec& weights, - const size_t minimumLeafSize) + const size_t minimumLeafSize, + const double minimumGainSplit) { // Clear children if needed. for (size_t i = 0; i < children.size(); ++i) @@ -533,7 +542,7 @@ void DecisionTree( @@ -556,7 +565,8 @@ void DecisionTreeTrain(data, currentChildBegin, currentCol - currentChildBegin, datasetInfo, labels, numClasses, - weights, currentCol - currentChildBegin); + weights, currentCol - currentChildBegin, minimumGainSplit); } else { child->Train(data, currentChildBegin, currentCol - currentChildBegin, datasetInfo, labels, numClasses, - weights, minimumLeafSize); + weights, minimumLeafSize, minimumGainSplit); } children.push_back(child); } @@ -685,7 +696,8 @@ void DecisionTree& labels, const size_t numClasses, arma::rowvec& weights, - const size_t minimumLeafSize) + const size_t minimumLeafSize, + const double minimumGainSplit) { // Clear children if needed. for (size_t i = 0; i < children.size(); ++i) @@ -717,7 +729,8 @@ void DecisionTree bestGain) { @@ -776,13 +789,13 @@ void DecisionTreeTrain(data, currentChildBegin, currentCol - currentChildBegin, labels, numClasses, weights, - currentCol - currentChildBegin); + currentCol - currentChildBegin, minimumGainSplit); } else { child->Train(data, currentChildBegin, currentCol - currentChildBegin, labels, numClasses, weights, - minimumLeafSize); + minimumLeafSize, minimumGainSplit); } children.push_back(child); } diff --git a/src/mlpack/methods/decision_tree/decision_tree_main.cpp b/src/mlpack/methods/decision_tree/decision_tree_main.cpp index e4ee8fcedc..da13a2691f 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_main.cpp +++ b/src/mlpack/methods/decision_tree/decision_tree_main.cpp @@ -39,7 +39,8 @@ PROGRAM_INFO("Decision tree", "may not be specified when the " + PRINT_PARAM_STRING("training") + " " "parameter is specified. The " + PRINT_PARAM_STRING("minimum_leaf_size") + " parameter specifies the minimum number of training points that must fall" - " into each leaf for it to be split. If " + + " into each leaf for it to be split. The " + PRINT_PARAM_STRING("minimum_gain_split") + + " parameter specifies the minimum gain that is needed for the node to split. If " + PRINT_PARAM_STRING("print_training_error") + " is specified, the training " "error will be printed." "\n\n" @@ -58,7 +59,7 @@ PROGRAM_INFO("Decision tree", "call" "\n\n" + PRINT_CALL("decision_tree", "training", "data", "labels", "labels", - "output_model", "tree", "minimum_leaf_size", 20, + "output_model", "tree", "minimum_leaf_size", 20, "minimum_gain_split", 1e-3, "print_training_error", true) + "\n\n" "Then, to use that model to classify points in " + @@ -82,6 +83,8 @@ PARAM_UMATRIX_IN("test_labels", "Test point labels, if accuracy calculation " // Training parameters. PARAM_INT_IN("minimum_leaf_size", "Minimum number of points in a leaf.", "n", 20); +PARAM_INT_IN("minimum_gain_split", "Minimum gain for node splitting.", "n", + 1e-3); PARAM_FLAG("print_training_error", "Print the training error.", "e"); // Output parameters. @@ -164,6 +167,7 @@ static void mlpackMain() // Now build the tree. const size_t minLeafSize = (size_t) CLI::GetParam("minimum_leaf_size"); + const size_t minimumGainSplit = (size_t) CLI::GetParam("minimum_gain_split"); // Create decision tree with weighted labels. if (CLI::HasParam("weights")) @@ -171,12 +175,12 @@ static void mlpackMain() arma::Row weights = std::move(CLI::GetParam>("weights")); model->tree = DecisionTree<>(trainingSet, model->info, labels, - numClasses, weights, minLeafSize); + numClasses, weights, minLeafSize, minimumGainSplit); } else { model->tree = DecisionTree<>(trainingSet, model->info, labels, - numClasses, minLeafSize); + numClasses, minLeafSize, minimumGainSplit); } // Do we need to print training error? From a222951c70ff145f236da637c3ad89030c32aa7f Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Tue, 6 Mar 2018 15:07:13 +0530 Subject: [PATCH 18/79] Default value for minimumGainSplit --- .../decision_tree/decision_tree_impl.hpp | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index 10b2a45cb1..a7c51f07bf 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -33,7 +33,7 @@ DecisionTree::type; using TrueLabelsType = typename std::decay::type; @@ -65,7 +65,7 @@ DecisionTree::type; using TrueLabelsType = typename std::decay::type; @@ -99,7 +99,7 @@ DecisionTree::type>::value>*) @@ -483,7 +483,7 @@ void DecisionTree::type>::value>*) @@ -533,7 +533,7 @@ void DecisionTree Date: Tue, 6 Mar 2018 21:50:30 +0530 Subject: [PATCH 19/79] Fix style issues, add test --- .../decision_tree/all_categorical_split.hpp | 4 +- .../all_categorical_split_impl.hpp | 4 +- .../best_binary_numeric_split.hpp | 4 +- .../best_binary_numeric_split_impl.hpp | 4 +- .../methods/decision_tree/decision_tree.hpp | 67 ++++++++++--------- .../decision_tree/decision_tree_impl.hpp | 32 ++++----- .../decision_tree/decision_tree_main.cpp | 3 + src/mlpack/tests/decision_tree_test.cpp | 63 ++++++++++++++--- 8 files changed, 113 insertions(+), 68 deletions(-) diff --git a/src/mlpack/methods/decision_tree/all_categorical_split.hpp b/src/mlpack/methods/decision_tree/all_categorical_split.hpp index 4116806fac..d2cd05001b 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split.hpp @@ -61,9 +61,9 @@ class AllCategoricalSplit const size_t numClasses, const WeightVecType& weights, const size_t minimumLeafSize, + const double minimumGainSplit, arma::Col& classProbabilities, - AuxiliarySplitInfo& aux, - const double minimumGainSplit); + AuxiliarySplitInfo& aux,); /** * Return the number of children in the split. diff --git a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp index a00a7b5957..115a9afb7e 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp @@ -25,9 +25,9 @@ double AllCategoricalSplit::SplitIfBetter( const size_t numClasses, const WeightVecType& weights, const size_t minimumLeafSize, + const double minimumGainSplit, arma::Col& classProbabilities, - AuxiliarySplitInfo& /* aux */, - const double minimumGainSplit) + AuxiliarySplitInfo& /* aux */) { // Count the number of elements in each potential child. const double epsilon = 1e-7; // Tolerance for floating-point errors. diff --git a/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp b/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp index 644145ec96..c187e5f08d 100644 --- a/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp +++ b/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp @@ -58,9 +58,9 @@ class BestBinaryNumericSplit const size_t numClasses, const WeightVecType& weights, const size_t minimumLeafSize, + const double minimumGainSplit, arma::Col& classProbabilities, - AuxiliarySplitInfo& aux, - const double minimumGainSplit); + AuxiliarySplitInfo& aux); /** * Returns 2, since the binary split always has two children. diff --git a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp b/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp index 13e99b5ea9..8641f1bf01 100644 --- a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp @@ -24,9 +24,9 @@ double BestBinaryNumericSplit::SplitIfBetter( const size_t numClasses, const WeightVecType& weights, const size_t minimumLeafSize, + const double minimumGainSplit, arma::Col& classProbabilities, - AuxiliarySplitInfo& /* aux */, - const double minimumGainSplit) + AuxiliarySplitInfo& /* aux */) { // First sanity check: if we don't have enough points, we can't split. if (data.n_elem < (minimumLeafSize * 2)) diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp index 7a4cacd0b9..a8108cdd94 100644 --- a/src/mlpack/methods/decision_tree/decision_tree.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree.hpp @@ -52,16 +52,16 @@ class DecisionTree : /** * Construct the decision tree on the given data and labels, where the data - * can be both numeric and categorical. Setting minimumLeafSize too small may - * cause the tree to overfit, but setting it too large may cause it to - * underfit. + * can be both numeric and categorical. Setting minimumLeafSize and + * minimumGainSplit too small may cause the tree to overfit, but setting them + * too large may cause it to underfit. * * @param data Dataset to train on. * @param datasetInfo Type information for each dimension of the dataset. * @param labels Labels for each training point. * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in each leaf node. - * @param minimumGainSplit Minimum Gain for the node to split. + * @param minimumGainSplit Minimum gain for the node to split. */ template DecisionTree(MatType&& data, @@ -73,15 +73,15 @@ class DecisionTree : /** * Construct the decision tree on the given data and labels, assuming that the - * data is all of the numeric type. Setting minimumLeafSize too small may - * cause the tree to overfit, but setting it too large may cause it to - * underfit. + * data is all of the numeric type. Setting minimumLeafSize and + * minimumGainSplit too small may cause the tree to overfit, but setting them + * too large may cause it to underfit. * * @param data Dataset to train on. * @param labels Labels for each training point. * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in each leaf node. - * @param minimumGainSplit Minimum Gain for the node to split. + * @param minimumGainSplit Minimum gain for the node to split. */ template DecisionTree(MatType&& data, @@ -92,9 +92,9 @@ class DecisionTree : /** * Construct the decision tree on the given data and labels with weights, - * where the data can be both numeric and categorical. Setting - * minimumLeafSize too small may cause the tree to overfit, but setting it too - * large may cause it to underfit. + * where the data can be both numeric and categorical. Setting minimumLeafSize + * and minimumGainSplit too small may cause the tree to overfit, but setting + * them too large may cause it to underfit. * * @param data Dataset to train on. * @param datasetInfo Type information for each dimension of the dataset. @@ -102,7 +102,7 @@ class DecisionTree : * @param numClasses Number of classes in the dataset. * @param weights The weight list of given label. * @param minimumLeafSize Minimum number of points in each leaf node. - * @param minimumGainSplit Minimum Gain for the node to split. + * @param minimumGainSplit Minimum gain for the node to split. */ template DecisionTree(MatType&& data, @@ -118,16 +118,16 @@ class DecisionTree : /** * Construct the decision tree on the given data and labels with weights, - * assuming that the data is all of the numeric type. Setting minimumLeafSize - * too small may cause the tree to overfit, but setting it too large may cause - * it to underfit. + * assuming that the data is all of the numeric type. Setting minimumLeafSize + * and minimumGainSplit too small may cause the tree to overfit, but setting + * them too large may cause it to underfit. * * @param data Dataset to train on. * @param labels Labels for each training point. * @param numClasses Number of classes in the dataset. * @param weights The Weight list of given labels. * @param minimumLeafSize Minimum number of points in each leaf node. - * @param minimumGainSplit Minimum Gain for the node to split. + * @param minimumGainSplit Minimum gain for the node to split. */ template DecisionTree(MatType&& data, @@ -187,8 +187,9 @@ class DecisionTree : /** * Train the decision tree on the given data. This will overwrite the * existing model. The data may have numeric and categorical types, specified - * by the datasetInfo parameter. Setting minimumLeafSize too small may cause - * the tree to overfit, but setting it too large may cause it to underfit. + * by the datasetInfo parameter. Setting minimumLeafSize and + * minimumGainSplit too small may cause the tree to overfit, but setting them + * too large may cause it to underfit. * * @param data Dataset to train on. * @param datasetInfo Type information for each dimension. @@ -196,7 +197,7 @@ class DecisionTree : * @param numClasses Number of classes in the dataset. * @param weights Weights of all the labels * @param minimumLeafSize Minimum number of points in each leaf node. - * @param minimumGainSplit Minimum Gain for the node to split. + * @param minimumGainSplit Minimum gain for the node to split. */ template void Train(MatType&& data, @@ -208,16 +209,16 @@ class DecisionTree : /** * Train the decision tree on the given data, assuming that all dimensions are - * numeric. This will overwrite the given model. Setting minimumLeafSize too - * small may cause the tree to overfit, but setting it too large may cause it - * to underfit. + * numeric. This will overwrite the given model. Setting minimumLeafSize and + * minimumGainSplit too small may cause the tree to overfit, but setting them + * too large may cause it to underfit. * * @param data Dataset to train on. * @param labels Labels for each training point. * @param numClasses Number of classes in the dataset. * @param weights Weights of all the labels * @param minimumLeafSize Minimum number of points in each leaf node. - * @param minimumGainSplit Minimum Gain for the node to split. + * @param minimumGainSplit Minimum gain for the node to split. */ template void Train(MatType&& data, @@ -229,9 +230,9 @@ class DecisionTree : /** * Train the decision tree on the given weighted data. This will overwrite * the existing model. The data may have numeric and categorical types, - * specified by the datasetInfo parameter. Setting minimumLeafSize too small - * may cause the tree to overfit, but setting it too large may cause it to - * underfit. + * specified by the datasetInfo parameter. Setting minimumLeafSize and + * minimumGainSplit too small may cause the tree to overfit, but setting them + * too large may cause it to underfit. * * @param data Dataset to train on. * @param datasetInfo Type information for each dimension. @@ -239,7 +240,7 @@ class DecisionTree : * @param numClasses Number of classes in the dataset. * @param weights Weights of all the labels * @param minimumLeafSize Minimum number of points in each leaf node. - * @param minimumGainSplit Minimum Gain for the node to split. + * @param minimumGainSplit Minimum gain for the node to split. */ template void Train(MatType&& data, @@ -254,16 +255,16 @@ class DecisionTree : /** * Train the decision tree on the given weighted data, assuming that all - * dimensions are numeric. This will overwrite the given model. Setting - * minimumLeafSize too small may cause the tree to overfit, but setting it too - * large may cause it to underfit. + * dimensions are numeric. This will overwrite the given model. Setting + * minimumLeafSize and minimumGainSplit too small may cause the tree to + * overfit, but setting them too large may cause it to underfit. * * @param data Dataset to train on. * @param labels Labels for each training point. * @param numClasses Number of classes in the dataset. * @param weights Weights of all the labels * @param minimumLeafSize Minimum number of points in each leaf node. - * @param minimumGainSplit Minimum Gain for the node to split. + * @param minimumGainSplit Minimum gain for the node to split. */ template void Train(MatType&& data, @@ -399,7 +400,7 @@ class DecisionTree : * @param labels Labels for each training point. * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in each leaf node. - * @param minimumGainSplit Minimum Gain for the node to split. + * @param minimumGainSplit Minimum gain for the node to split. */ template void Train(MatType& data, @@ -424,7 +425,7 @@ class DecisionTree : * @param labels Labels for each training point. * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in each leaf node. - * @param minimumGainSplit Minimum Gain for the node to split. + * @param minimumGainSplit Minimum gain for the node to split. */ template void Train(MatType& data, diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index a7c51f07bf..9e9a971c4e 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -33,7 +33,7 @@ DecisionTree::type; using TrueLabelsType = typename std::decay::type; @@ -65,7 +65,7 @@ DecisionTree::type; using TrueLabelsType = typename std::decay::type; @@ -99,7 +99,7 @@ DecisionTree::type>::value>*) @@ -483,7 +483,7 @@ void DecisionTree::type>::value>*) @@ -533,7 +533,7 @@ void DecisionTree bestGain) { diff --git a/src/mlpack/methods/decision_tree/decision_tree_main.cpp b/src/mlpack/methods/decision_tree/decision_tree_main.cpp index da13a2691f..ea63fe9df2 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_main.cpp +++ b/src/mlpack/methods/decision_tree/decision_tree_main.cpp @@ -139,6 +139,9 @@ static void mlpackMain() RequireParamValue("minimum_leaf_size", [](int x) { return x > 0; }, true, "leaf size must be positive"); + RequireParamValue("minimum_gain_split", [](int x) { return x > 0 && x < 1; }, true, + "leaf size must be a fraction in range [0,1]"); + // Load the model or build the tree. DecisionTreeModel* model; arma::mat trainingSet; diff --git a/src/mlpack/tests/decision_tree_test.cpp b/src/mlpack/tests/decision_tree_test.cpp index 63f42f6324..2bddf353f2 100644 --- a/src/mlpack/tests/decision_tree_test.cpp +++ b/src/mlpack/tests/decision_tree_test.cpp @@ -288,10 +288,10 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitSimpleSplitTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 2, weights); const double gain = BestBinaryNumericSplit::SplitIfBetter( - bestGain, values, labels, 2, weights, 3, classProbabilities, aux); + bestGain, values, labels, 2, weights, 3, 1e-7, classProbabilities, aux); const double weightedGain = BestBinaryNumericSplit::SplitIfBetter(bestGain, values, - labels, 2, weights, 3, classProbabilities, aux); + labels, 2, weights, 3, 1e-7, classProbabilities, aux); // Make sure that a split was made. BOOST_REQUIRE_GT(gain, bestGain); @@ -325,11 +325,11 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitMinSamplesTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 2, weights); const double gain = BestBinaryNumericSplit::SplitIfBetter( - bestGain, values, labels, 2, weights, 8, classProbabilities, aux); + bestGain, values, labels, 2, weights, 8, 1e-7, classProbabilities, aux); // This should make no difference because it won't split at all. const double weightedGain = BestBinaryNumericSplit::SplitIfBetter(bestGain, values, - labels, 2, weights, 8, classProbabilities, aux); + labels, 2, weights, 8, 1e-7, classProbabilities, aux); // Make sure that no split was made. BOOST_REQUIRE_EQUAL(gain, bestGain); @@ -360,7 +360,7 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitNoGainTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 2, weights); const double gain = BestBinaryNumericSplit::SplitIfBetter( - bestGain, values, labels, 2, weights, 10, classProbabilities, aux); + bestGain, values, labels, 2, weights, 10, 1e-7, classProbabilities, aux); // Make sure there was no split. BOOST_REQUIRE_EQUAL(gain, bestGain); @@ -384,10 +384,10 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitSimpleSplitTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); const double gain = AllCategoricalSplit::SplitIfBetter( - bestGain, values, 4, labels, 3, weights, 3, classProbabilities, aux); + bestGain, values, 4, labels, 3, weights, 3, 1e-7, classProbabilities, aux); const double weightedGain = AllCategoricalSplit::SplitIfBetter(bestGain, values, 4, - labels, 3, weights, 3, classProbabilities, aux); + labels, 3, weights, 3, 1e-7, classProbabilities, aux); // Make sure that a split was made. BOOST_REQUIRE_GT(gain, bestGain); @@ -419,7 +419,7 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitMinSamplesTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); const double gain = AllCategoricalSplit::SplitIfBetter( - bestGain, values, 4, labels, 3, weights, 4, classProbabilities, aux); + bestGain, values, 4, labels, 3, weights, 4, 1e-7, classProbabilities, aux); // Make sure it's not split. BOOST_REQUIRE_EQUAL(gain, bestGain); @@ -451,10 +451,10 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitNoGainTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); const double gain = AllCategoricalSplit::SplitIfBetter( - bestGain, values, 10, labels, 3, weights, 10, classProbabilities, aux); + bestGain, values, 10, labels, 3, weights, 10, 1e-7, classProbabilities, aux); const double weightedGain = AllCategoricalSplit::SplitIfBetter(bestGain, values, 10, - labels, 3, weights, 10, classProbabilities, aux); + labels, 3, weights, 10, 1e-7, classProbabilities, aux); // Make sure that there was no split. BOOST_REQUIRE_EQUAL(gain, bestGain); @@ -539,7 +539,7 @@ BOOST_AUTO_TEST_CASE(PerfectTrainingSet) } /** - * onstruct the decision tree with weighted labels + * Construct the decision tree with weighted labels */ BOOST_AUTO_TEST_CASE(PerfectTrainingSetWithWeight) { @@ -1082,4 +1082,45 @@ BOOST_AUTO_TEST_CASE(ConstDataTest) constWeights); } +/** + * Construct the decision tree with splitting only if gain is more than + * threshold. + */ +BOOST_AUTO_TEST_CASE(RegularisedDecisionTree) +{ + // Completely random dataset with no structure. + arma::mat dataset(10, 1000, arma::fill::randu); + arma::Row labels(1000); + for (size_t i = 0; i < 1000; ++i) + labels[i] = i % 3; // 3 classes. + arma::rowvec weights(labels.n_elem); + weights.ones(); + + // Minimum leaf size of 1. + DecisionTree<> d(dataset, labels, 3, weights, 1, 1e-7); + + // Minimum leaf size of 1 and Minimum gain split of 0.01. + DecisionTree<> dRegularised(dataset, labels, 3, weights, 1, 0.01); + + size_t count = 0; + // This part of code is dupliacte with no weighted one. + for (size_t i = 0; i < 1000; ++i) + { + size_t prediction, predictionsregularised; + arma::vec probabilities, probabilitiesRegularised; + + d.Classify(dataset.col(i), prediction, probabilities); + dRegularised.Classify(dataset.col(i), predictionsregularised, + probabilitiesRegularised); + + if(prediction != predictionsregularised) + count++; + + BOOST_REQUIRE_EQUAL(probabilities.n_elem, 3); + BOOST_REQUIRE_EQUAL(probabilitiesRegularised.n_elem, 3); + } + + BOOST_REQUIRE_GT(count, 0); +} + BOOST_AUTO_TEST_SUITE_END(); From cc0b9187d0cad7c607e47108de09f5f323a6643f Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Tue, 6 Mar 2018 22:31:52 +0530 Subject: [PATCH 20/79] Minor fix --- src/mlpack/methods/decision_tree/all_categorical_split.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/decision_tree/all_categorical_split.hpp b/src/mlpack/methods/decision_tree/all_categorical_split.hpp index d2cd05001b..2823fa9bb5 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split.hpp @@ -63,7 +63,7 @@ class AllCategoricalSplit const size_t minimumLeafSize, const double minimumGainSplit, arma::Col& classProbabilities, - AuxiliarySplitInfo& aux,); + AuxiliarySplitInfo& aux); /** * Return the number of children in the split. From 08a9b8f1af64ee6cd0902de1d874d64e982d7d73 Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Wed, 7 Mar 2018 01:51:46 +0530 Subject: [PATCH 21/79] Solve alias issue --- src/mlpack/methods/decision_tree/decision_tree_main.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/decision_tree/decision_tree_main.cpp b/src/mlpack/methods/decision_tree/decision_tree_main.cpp index ea63fe9df2..1fcea6b652 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_main.cpp +++ b/src/mlpack/methods/decision_tree/decision_tree_main.cpp @@ -83,8 +83,8 @@ PARAM_UMATRIX_IN("test_labels", "Test point labels, if accuracy calculation " // Training parameters. PARAM_INT_IN("minimum_leaf_size", "Minimum number of points in a leaf.", "n", 20); -PARAM_INT_IN("minimum_gain_split", "Minimum gain for node splitting.", "n", - 1e-3); +PARAM_INT_IN("minimum_gain_split", "Minimum gain for node splitting.", "g", + 1e-7); PARAM_FLAG("print_training_error", "Print the training error.", "e"); // Output parameters. From 98ac1dbdc5aa02faf88c7e0b905a94d38f3a2964 Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Wed, 7 Mar 2018 10:53:33 +0530 Subject: [PATCH 22/79] Test for CLI --- .../decision_tree/decision_tree_main.cpp | 17 ++++++----- src/mlpack/tests/decision_tree_test.cpp | 11 ++++--- .../tests/main_tests/decision_tree_test.cpp | 29 +++++++++++++++++++ 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/src/mlpack/methods/decision_tree/decision_tree_main.cpp b/src/mlpack/methods/decision_tree/decision_tree_main.cpp index 1fcea6b652..0f5f27c820 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_main.cpp +++ b/src/mlpack/methods/decision_tree/decision_tree_main.cpp @@ -39,8 +39,9 @@ PROGRAM_INFO("Decision tree", "may not be specified when the " + PRINT_PARAM_STRING("training") + " " "parameter is specified. The " + PRINT_PARAM_STRING("minimum_leaf_size") + " parameter specifies the minimum number of training points that must fall" - " into each leaf for it to be split. The " + PRINT_PARAM_STRING("minimum_gain_split") + - " parameter specifies the minimum gain that is needed for the node to split. If " + + " into each leaf for it to be split. The " + + PRINT_PARAM_STRING("minimum_gain_split") + " parameter specifies " + "the minimum gain that is needed for the node to split. If " + PRINT_PARAM_STRING("print_training_error") + " is specified, the training " "error will be printed." "\n\n" @@ -59,8 +60,8 @@ PROGRAM_INFO("Decision tree", "call" "\n\n" + PRINT_CALL("decision_tree", "training", "data", "labels", "labels", - "output_model", "tree", "minimum_leaf_size", 20, "minimum_gain_split", 1e-3, - "print_training_error", true) + + "output_model", "tree", "minimum_leaf_size", 20, "minimum_gain_split", + 1e-3, "print_training_error", true) + "\n\n" "Then, to use that model to classify points in " + PRINT_DATASET("test_set") + " and print the test error given the " @@ -139,8 +140,9 @@ static void mlpackMain() RequireParamValue("minimum_leaf_size", [](int x) { return x > 0; }, true, "leaf size must be positive"); - RequireParamValue("minimum_gain_split", [](int x) { return x > 0 && x < 1; }, true, - "leaf size must be a fraction in range [0,1]"); + RequireParamValue("minimum_gain_split", [](double x) + { return x > 0.0 && x < 1.0; }, true, + "gain split must be a fraction in range [0,1]"); // Load the model or build the tree. DecisionTreeModel* model; @@ -170,7 +172,8 @@ static void mlpackMain() // Now build the tree. const size_t minLeafSize = (size_t) CLI::GetParam("minimum_leaf_size"); - const size_t minimumGainSplit = (size_t) CLI::GetParam("minimum_gain_split"); + const size_t minimumGainSplit = + (size_t) CLI::GetParam("minimum_gain_split"); // Create decision tree with weighted labels. if (CLI::HasParam("weights")) diff --git a/src/mlpack/tests/decision_tree_test.cpp b/src/mlpack/tests/decision_tree_test.cpp index 2bddf353f2..b2b45abd0f 100644 --- a/src/mlpack/tests/decision_tree_test.cpp +++ b/src/mlpack/tests/decision_tree_test.cpp @@ -384,7 +384,8 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitSimpleSplitTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); const double gain = AllCategoricalSplit::SplitIfBetter( - bestGain, values, 4, labels, 3, weights, 3, 1e-7, classProbabilities, aux); + bestGain, values, 4, labels, 3, weights, 3, 1e-7, classProbabilities, + aux); const double weightedGain = AllCategoricalSplit::SplitIfBetter(bestGain, values, 4, labels, 3, weights, 3, 1e-7, classProbabilities, aux); @@ -419,7 +420,8 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitMinSamplesTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); const double gain = AllCategoricalSplit::SplitIfBetter( - bestGain, values, 4, labels, 3, weights, 4, 1e-7, classProbabilities, aux); + bestGain, values, 4, labels, 3, weights, 4, 1e-7, classProbabilities, + aux); // Make sure it's not split. BOOST_REQUIRE_EQUAL(gain, bestGain); @@ -451,7 +453,8 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitNoGainTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); const double gain = AllCategoricalSplit::SplitIfBetter( - bestGain, values, 10, labels, 3, weights, 10, 1e-7, classProbabilities, aux); + bestGain, values, 10, labels, 3, weights, 10, 1e-7, classProbabilities, + aux); const double weightedGain = AllCategoricalSplit::SplitIfBetter(bestGain, values, 10, labels, 3, weights, 10, 1e-7, classProbabilities, aux); @@ -1113,7 +1116,7 @@ BOOST_AUTO_TEST_CASE(RegularisedDecisionTree) dRegularised.Classify(dataset.col(i), predictionsregularised, probabilitiesRegularised); - if(prediction != predictionsregularised) + if (prediction != predictionsregularised) count++; BOOST_REQUIRE_EQUAL(probabilities.n_elem, 3); diff --git a/src/mlpack/tests/main_tests/decision_tree_test.cpp b/src/mlpack/tests/main_tests/decision_tree_test.cpp index 1d47b4c127..944c48d4b5 100644 --- a/src/mlpack/tests/main_tests/decision_tree_test.cpp +++ b/src/mlpack/tests/main_tests/decision_tree_test.cpp @@ -167,6 +167,35 @@ BOOST_AUTO_TEST_CASE(DecisionTreeMinimumLeafSizeTest) Log::Fatal.ignoreInput = false; } +/** + * Make sure minimum gain split is always a fraction in range [0,1]. + */ +BOOST_AUTO_TEST_CASE(DecisionMinimumGainSplitTest) +{ + arma::mat inputData; + DatasetInfo info; + if (!data::Load("braziltourism.arff", inputData, info)) + BOOST_FAIL("Cannot load train dataset braziltourism.arff!"); + + arma::Row labels; + if (!data::Load("braziltourism_labels.txt", labels)) + BOOST_FAIL("Cannot load labels for braziltourism_labels.txt"); + + // Initialize an all-ones weight matrix. + arma::mat weights(1, labels.n_cols, arma::fill::ones); + + // Input training data. + SetInputParam("training", std::move(std::make_tuple(info, inputData))); + SetInputParam("labels", std::move(labels)); + SetInputParam("weights", std::move(weights)); + + SetInputParam("minimum_gain_split", 1.5); // Invalid. + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + /** * Ensure that saved model can be used again. */ From 67641de6aedd4b44de061bbed6ca1874630a0d05 Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Wed, 7 Mar 2018 19:44:58 +0530 Subject: [PATCH 23/79] Resolve typecast error for CLI test --- .../methods/decision_tree/decision_tree_main.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/decision_tree/decision_tree_main.cpp b/src/mlpack/methods/decision_tree/decision_tree_main.cpp index 0f5f27c820..258689fb6f 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_main.cpp +++ b/src/mlpack/methods/decision_tree/decision_tree_main.cpp @@ -84,7 +84,7 @@ PARAM_UMATRIX_IN("test_labels", "Test point labels, if accuracy calculation " // Training parameters. PARAM_INT_IN("minimum_leaf_size", "Minimum number of points in a leaf.", "n", 20); -PARAM_INT_IN("minimum_gain_split", "Minimum gain for node splitting.", "g", +PARAM_DOUBLE_IN("minimum_gain_split", "Minimum gain for node splitting.", "g", 1e-7); PARAM_FLAG("print_training_error", "Print the training error.", "e"); @@ -140,8 +140,8 @@ static void mlpackMain() RequireParamValue("minimum_leaf_size", [](int x) { return x > 0; }, true, "leaf size must be positive"); - RequireParamValue("minimum_gain_split", [](double x) - { return x > 0.0 && x < 1.0; }, true, + RequireParamValue("minimum_gain_split", [](double x) + { return (x > 0.0 && x < 1.0); }, true, "gain split must be a fraction in range [0,1]"); // Load the model or build the tree. @@ -172,8 +172,8 @@ static void mlpackMain() // Now build the tree. const size_t minLeafSize = (size_t) CLI::GetParam("minimum_leaf_size"); - const size_t minimumGainSplit = - (size_t) CLI::GetParam("minimum_gain_split"); + const double minimumGainSplit = + (double) CLI::GetParam("minimum_gain_split"); // Create decision tree with weighted labels. if (CLI::HasParam("weights")) From dfd07c0601543b591bfd6d1e8eadc89896c000c3 Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Fri, 9 Mar 2018 20:49:57 +0530 Subject: [PATCH 24/79] CLI test for minimum_gain_split --- .../tests/main_tests/decision_tree_test.cpp | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/mlpack/tests/main_tests/decision_tree_test.cpp b/src/mlpack/tests/main_tests/decision_tree_test.cpp index 944c48d4b5..c6a7ba067f 100644 --- a/src/mlpack/tests/main_tests/decision_tree_test.cpp +++ b/src/mlpack/tests/main_tests/decision_tree_test.cpp @@ -196,6 +196,61 @@ BOOST_AUTO_TEST_CASE(DecisionMinimumGainSplitTest) Log::Fatal.ignoreInput = false; } +/** + * Make sure minimum gain split produces regularised tree. + */ +BOOST_AUTO_TEST_CASE(DecisionRegularisationTest) +{ + // Completely random dataset with no structure. + arma::mat dataset(10, 1000, arma::fill::randu); + arma::mat pred, predRegularised; + arma::Row labels(1000); + for (size_t i = 0; i < 1000; ++i) + labels[i] = i % 3; // 3 classes. + arma::rowvec weights(labels.n_elem); + weights.ones(); + + // Input training data. + SetInputParam("training", dataset); + SetInputParam("labels", labels); + SetInputParam("weights", weights); + + SetInputParam("minimum_gain_split", 1e-7); + + // Input test data. + SetInputParam("test", dataset); + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; + pred = std::move(CLI::GetParam>("predictions")); + + // Input training data. + SetInputParam("training", dataset); + SetInputParam("labels", std::move(labels)); + SetInputParam("weights", std::move(weights)); + + SetInputParam("minimum_gain_split", 0.01); + + // Input test data. + SetInputParam("test", std::move(dataset)); + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; + predRegularised = std::move(CLI::GetParam>("predictions")); + + size_t count = 0; + // This part of code is dupliacte with no weighted one. + for (size_t i = 0; i < 1000; ++i) + { + if (pred[i] != predRegularised[i]) + count++; + } + + BOOST_REQUIRE_GT(count, 0); +} + /** * Ensure that saved model can be used again. */ From 1593d64dab228730af26354257c1b61c5eabe5c5 Mon Sep 17 00:00:00 2001 From: Moksh Jain Date: Fri, 9 Mar 2018 21:34:19 +0530 Subject: [PATCH 25/79] rename adamirror to optimisticadam --- .../core/optimizers/adam/CMakeLists.txt | 2 +- src/mlpack/core/optimizers/adam/adam.hpp | 4 ++-- ...r_update.hpp => optimisticadam_update.hpp} | 24 +++++++++---------- src/mlpack/tests/adam_test.cpp | 14 +++++------ 4 files changed, 22 insertions(+), 22 deletions(-) rename src/mlpack/core/optimizers/adam/{adamirror_update.hpp => optimisticadam_update.hpp} (84%) diff --git a/src/mlpack/core/optimizers/adam/CMakeLists.txt b/src/mlpack/core/optimizers/adam/CMakeLists.txt index 908fe06db9..fd5788618b 100644 --- a/src/mlpack/core/optimizers/adam/CMakeLists.txt +++ b/src/mlpack/core/optimizers/adam/CMakeLists.txt @@ -6,7 +6,7 @@ set(SOURCES amsgrad_update.hpp nadam_update.hpp nadamax_update.hpp - adamirror_update.hpp + optimisticadam_update.hpp ) set(DIR_SRCS) diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp index f0c209cf01..e1fa7d6df8 100644 --- a/src/mlpack/core/optimizers/adam/adam.hpp +++ b/src/mlpack/core/optimizers/adam/adam.hpp @@ -30,7 +30,7 @@ #include "amsgrad_update.hpp" #include "nadam_update.hpp" #include "nadamax_update.hpp" -#include "adamirror_update.hpp" +#include "optimisticadam_update.hpp" namespace mlpack { namespace optimization { @@ -187,7 +187,7 @@ using Nadam = AdamType; using NadaMax = AdamType; -using Adamirror = AdamType; +using OptimisticAdam = AdamType; } // namespace optimization } // namespace mlpack diff --git a/src/mlpack/core/optimizers/adam/adamirror_update.hpp b/src/mlpack/core/optimizers/adam/optimisticadam_update.hpp similarity index 84% rename from src/mlpack/core/optimizers/adam/adamirror_update.hpp rename to src/mlpack/core/optimizers/adam/optimisticadam_update.hpp index 1e934c3b75..4bbed2ab33 100644 --- a/src/mlpack/core/optimizers/adam/adamirror_update.hpp +++ b/src/mlpack/core/optimizers/adam/optimisticadam_update.hpp @@ -1,8 +1,8 @@ /** - * @file adamirror_update.hpp + * @file optimisticadam_update.hpp * @author Moksh Jain * - * Adamirror optimizer. Optimistic Adam is an an algorithm which uses + * OptmisticAdam optimizer. Optimistic Adam is an an algorithm which uses * Optimistic Mirror Descent with the Adam optimizer. * * mlpack is free software; you may redistribute it and/or modify it under the @@ -10,8 +10,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_OPTIMIZERS_ADAM_ADAMIRROR_UPDATE_HPP -#define MLPACK_CORE_OPTIMIZERS_ADAM_ADAMIRROR_UPDATE_HPP +#ifndef MLPACK_CORE_OPTIMIZERS_ADAM_OPTIMISTICADAM_UPDATE_HPP +#define MLPACK_CORE_OPTIMIZERS_ADAM_OPTIMISTICADAM_UPDATE_HPP #include @@ -19,7 +19,7 @@ namespace mlpack { namespace optimization { /** - * Adamirror is an optimizer which uses Optmistic Mirror Descent with + * OptimisticAdam is an optimizer which uses Optmistic Mirror Descent with * the Adam Optimizer. * * For more information, see the following. @@ -34,20 +34,20 @@ namespace optimization { * } * @endcode */ -class AdamirrorUpdate +class OptimisticAdamUpdate { public: /** - * Construct the Adamirror update policy with the given parameters. + * Construct the OptimisticAdam update policy with the given parameters. * * @param epsilon The epsilon value used to initialise the squared gradient * parameter. * @param beta1 The smoothing parameter. * @param beta2 The second moment coefficient. */ - AdamirrorUpdate(const double epsilon = 1e-8, - const double beta1 = 0.9, - const double beta2 = 0.999) : + OptimisticAdamUpdate(const double epsilon = 1e-8, + const double beta1 = 0.9, + const double beta2 = 0.999) : epsilon(epsilon), beta1(beta1), beta2(beta2), @@ -71,7 +71,7 @@ class AdamirrorUpdate } /** - * Update step for Adamirror. + * Update step for OptimisticAdam. * * @param iterate Parameters that minimize the function. * @param stepSize Step size to be used for the given iteration. @@ -98,7 +98,7 @@ class AdamirrorUpdate iterate -= (2 * stepSize * update - stepSize * g); - g = update; + g = std::move(update); } //! Get the value used to initialise the squared gradient parameter. diff --git a/src/mlpack/tests/adam_test.cpp b/src/mlpack/tests/adam_test.cpp index 279c8e5ffd..dff4844db3 100644 --- a/src/mlpack/tests/adam_test.cpp +++ b/src/mlpack/tests/adam_test.cpp @@ -509,12 +509,12 @@ BOOST_AUTO_TEST_CASE(NadaMaxLogisticRegressionTest) } /** - * Tests the Adamirror optimizer using a simple test function. + * Tests the OptimisticAdam optimizer using a simple test function. */ -BOOST_AUTO_TEST_CASE(SimpleAdamirrorTestFunction) +BOOST_AUTO_TEST_CASE(SimpleOptimisticAdamTestFunction) { SGDTestFunction f; - Adamirror optimizer(1e-2, 1, 0.9, 0.99, 1e-8); + OptimisticAdam optimizer(1e-2, 1, 0.9, 0.99, 1e-8); arma::mat coordinates = f.GetInitialPoint(); optimizer.Optimize(f, coordinates); @@ -525,9 +525,9 @@ BOOST_AUTO_TEST_CASE(SimpleAdamirrorTestFunction) } /** - * Run Adamirror on logistic regression and make sure the results are acceptable. + * Run OptimisticAdam on logistic regression and make sure the results are acceptable. */ -BOOST_AUTO_TEST_CASE(AdamirrorLogisticRegressionTest) +BOOST_AUTO_TEST_CASE(OptimisticAdamLogisticRegressionTest) { // Generate a two-Gaussian dataset. GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), @@ -573,8 +573,8 @@ BOOST_AUTO_TEST_CASE(AdamirrorLogisticRegressionTest) testResponses[i] = 1; } - Adamirror adamirror; - LogisticRegression<> lr(shuffledData, shuffledResponses, adamirror, 0.5); + OptimisticAdam optimisticAdam; + LogisticRegression<> lr(shuffledData, shuffledResponses, optimisticAdam, 0.5); // Ensure that the error is close to zero. const double acc = lr.ComputeAccuracy(data, responses); From 833f5f577baa3f926e156a3722db0017ed0936f1 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Fri, 9 Mar 2018 21:54:56 +0530 Subject: [PATCH 26/79] Modified method rule --- src/mlpack/core/optimizers/sgd/sgd.hpp | 2 +- .../nesterov_momentum_update.hpp | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/sgd.hpp b/src/mlpack/core/optimizers/sgd/sgd.hpp index 523539ec73..047e871048 100644 --- a/src/mlpack/core/optimizers/sgd/sgd.hpp +++ b/src/mlpack/core/optimizers/sgd/sgd.hpp @@ -107,7 +107,7 @@ class SGD * @param resetPolicy Flag that determines whether update policy parameters * are reset before every Optimize call. */ - SGD(const double stepSize = 0.01, + SGD(const double stepSize = 0.001, const size_t batchSize = 32, const size_t maxIterations = 100000, const double tolerance = 1e-5, diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 96f01b5824..c79f58e739 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -43,7 +43,9 @@ class NesterovMomentumUpdate * Construct the Nesterov Momentum update policy with the given parameters. * */ - NesterovMomentumUpdate() : iteration(0) + NesterovMomentumUpdate(const double maxMomentum = 0.999) : + iteration(0), + maxMomentum(maxMomentum) { // Nothing to do. } @@ -78,19 +80,28 @@ class NesterovMomentumUpdate { iteration++; - double momentum = 1 - (3 / (iteration + 5)); + double momentumT = std::min((1 - std::pow(2,(- 1 - ((log(floor(iteration + / 250)) +1) / log(2))))) , maxMomentum); - velocity = momentum * velocity - stepSize * gradient; + velocity = momentumT * velocity - stepSize * gradient; iterate += velocity; } + //! Get the value used to initialise the maximum momentum coefficient. + double MaxMomentum() const { return maxMomentum; } + //! Modify the value used to initialise the maximum momentum coefficient. + double& MaxMomentum() { return maxMomentum; } + private: // The velocity matrix. arma::mat velocity; // The number of iterations. double iteration; + + // Maximum momentum coefficient + double maxMomentum; }; } // namespace optimization From 1340c79f637fd0d0438f6fad9f6d68eef5217859 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Fri, 9 Mar 2018 21:58:31 +0530 Subject: [PATCH 27/79] Modified test --- src/mlpack/tests/nesterov_momentum_sgd_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp index 18ab96ae0b..442fa80ed7 100644 --- a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp +++ b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp @@ -31,7 +31,7 @@ BOOST_AUTO_TEST_SUITE(NesterovMomentumSGDTest); BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) { SGDTestFunction f; - NesterovMomentumUpdate nesterovMomentumUpdate; + NesterovMomentumUpdate nesterovMomentumUpdate(0.999); NesterovMomentumSGD s(0.0003, 1, 2500000, 1e-9, true, nesterovMomentumUpdate); @@ -66,7 +66,7 @@ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) { // Create the generalized Rosenbrock function. GeneralizedRosenbrockFunction f(i); - NesterovMomentumUpdate nesterovMomentumUpdate; + NesterovMomentumUpdate nesterovMomentumUpdate(0.999); NesterovMomentumSGD s(0.0008, 1, 0, 1e-15, true, nesterovMomentumUpdate); arma::mat coordinates = f.GetInitialPoint(); From 5f2baa73d44353c0225a5c97b507c2489ac30b14 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Fri, 9 Mar 2018 22:01:59 +0530 Subject: [PATCH 28/79] Style fix --- .../optimizers/sgd/update_policies/nesterov_momentum_update.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index c79f58e739..75eea974ca 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -80,7 +80,7 @@ class NesterovMomentumUpdate { iteration++; - double momentumT = std::min((1 - std::pow(2,(- 1 - ((log(floor(iteration + double momentumT = std::min((1 - std::pow(2 , (- 1 - ((log(floor(iteration / 250)) +1) / log(2))))) , maxMomentum); velocity = momentumT * velocity - stepSize * gradient; From 66c7ac481bd9ba67cde5587df69b302dca719bd9 Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Sat, 10 Mar 2018 01:08:50 +0530 Subject: [PATCH 29/79] Update CLI test --- src/mlpack/tests/main_tests/decision_tree_test.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/mlpack/tests/main_tests/decision_tree_test.cpp b/src/mlpack/tests/main_tests/decision_tree_test.cpp index c6a7ba067f..5ea4c906ff 100644 --- a/src/mlpack/tests/main_tests/decision_tree_test.cpp +++ b/src/mlpack/tests/main_tests/decision_tree_test.cpp @@ -220,9 +220,7 @@ BOOST_AUTO_TEST_CASE(DecisionRegularisationTest) // Input test data. SetInputParam("test", dataset); - Log::Fatal.ignoreInput = true; - BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); - Log::Fatal.ignoreInput = false; + mlpackMain(); pred = std::move(CLI::GetParam>("predictions")); // Input training data. @@ -235,9 +233,7 @@ BOOST_AUTO_TEST_CASE(DecisionRegularisationTest) // Input test data. SetInputParam("test", std::move(dataset)); - Log::Fatal.ignoreInput = true; - BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); - Log::Fatal.ignoreInput = false; + mlpackMain(); predRegularised = std::move(CLI::GetParam>("predictions")); size_t count = 0; From 6837ef2b657ebd84ada29b1828cc573b113f124b Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Sat, 10 Mar 2018 11:02:57 +0530 Subject: [PATCH 30/79] Fix build errors --- src/mlpack/tests/main_tests/decision_tree_test.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/mlpack/tests/main_tests/decision_tree_test.cpp b/src/mlpack/tests/main_tests/decision_tree_test.cpp index 5ea4c906ff..1b8403a8fe 100644 --- a/src/mlpack/tests/main_tests/decision_tree_test.cpp +++ b/src/mlpack/tests/main_tests/decision_tree_test.cpp @@ -203,7 +203,6 @@ BOOST_AUTO_TEST_CASE(DecisionRegularisationTest) { // Completely random dataset with no structure. arma::mat dataset(10, 1000, arma::fill::randu); - arma::mat pred, predRegularised; arma::Row labels(1000); for (size_t i = 0; i < 1000; ++i) labels[i] = i % 3; // 3 classes. @@ -219,7 +218,7 @@ BOOST_AUTO_TEST_CASE(DecisionRegularisationTest) // Input test data. SetInputParam("test", dataset); - + arma::Row pred; mlpackMain(); pred = std::move(CLI::GetParam>("predictions")); @@ -232,7 +231,7 @@ BOOST_AUTO_TEST_CASE(DecisionRegularisationTest) // Input test data. SetInputParam("test", std::move(dataset)); - + arma::Row predRegularised; mlpackMain(); predRegularised = std::move(CLI::GetParam>("predictions")); From 5c72999fc283a2ce91d3e24b3ba4bc238a52e069 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Sat, 10 Mar 2018 15:10:25 +0530 Subject: [PATCH 31/79] Tried tests for constant momentum --- .../sgd/update_policies/nesterov_momentum_update.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 75eea974ca..54a55224ef 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -80,8 +80,7 @@ class NesterovMomentumUpdate { iteration++; - double momentumT = std::min((1 - std::pow(2 , (- 1 - ((log(floor(iteration - / 250)) +1) / log(2))))) , maxMomentum); + double momentumT = 0.9; velocity = momentumT * velocity - stepSize * gradient; From 9d6ac4d6e77143c4a929cc91969c1a1105335d0e Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Sat, 10 Mar 2018 22:37:44 +0530 Subject: [PATCH 32/79] Changed momentum to check whether tests are running correctly --- .../optimizers/sgd/update_policies/nesterov_momentum_update.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 54a55224ef..8a2dfda640 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -80,7 +80,7 @@ class NesterovMomentumUpdate { iteration++; - double momentumT = 0.9; + double momentumT = 0.5; velocity = momentumT * velocity - stepSize * gradient; From e65e757d39ad7bae5bac08f9708a031d1adaa030 Mon Sep 17 00:00:00 2001 From: Moksh Jain Date: Sun, 11 Mar 2018 10:37:34 +0530 Subject: [PATCH 33/79] indentation fix --- src/mlpack/core/optimizers/adam/optimisticadam_update.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/optimizers/adam/optimisticadam_update.hpp b/src/mlpack/core/optimizers/adam/optimisticadam_update.hpp index 4bbed2ab33..27e0dd8484 100644 --- a/src/mlpack/core/optimizers/adam/optimisticadam_update.hpp +++ b/src/mlpack/core/optimizers/adam/optimisticadam_update.hpp @@ -46,8 +46,8 @@ class OptimisticAdamUpdate * @param beta2 The second moment coefficient. */ OptimisticAdamUpdate(const double epsilon = 1e-8, - const double beta1 = 0.9, - const double beta2 = 0.999) : + const double beta1 = 0.9, + const double beta2 = 0.999) : epsilon(epsilon), beta1(beta1), beta2(beta2), From aff6a762c3a71cc9f591e5a8102c6e3ef1c8bd7e Mon Sep 17 00:00:00 2001 From: Wenhao Huang Date: Sun, 11 Mar 2018 13:46:38 +0800 Subject: [PATCH 34/79] add SetFixedRandomSeed --- src/mlpack/core/math/random.hpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/mlpack/core/math/random.hpp b/src/mlpack/core/math/random.hpp index 6d615f4a92..c30c011660 100644 --- a/src/mlpack/core/math/random.hpp +++ b/src/mlpack/core/math/random.hpp @@ -48,6 +48,19 @@ inline void RandomSeed(const size_t seed) #endif } +/** + * Set the random seed to a predefined seed. + */ +#if (BINDING_TYPE == BINDING_TYPE_TEST) +inline void SetFixedRandomSeed() +{ + const size_t seed = 54321; + randGen.seed((uint32_t) seed); + srand((unsigned int) seed); + arma::arma_rng::set_seed(seed); +} +#endif + /** * Generates a uniform random number between 0 and 1. */ From c11f5f8a7b664ba22cfc0c60e6f68bd7e2526336 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Sun, 11 Mar 2018 20:23:16 +0530 Subject: [PATCH 35/79] Formula Correction --- .../nesterov_momentum_update.hpp | 27 +++++++------------ .../tests/nesterov_momentum_sgd_test.cpp | 4 +-- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 8a2dfda640..5616fd68cd 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -43,9 +43,9 @@ class NesterovMomentumUpdate * Construct the Nesterov Momentum update policy with the given parameters. * */ - NesterovMomentumUpdate(const double maxMomentum = 0.999) : + NesterovMomentumUpdate(const double momentum = 0.5) : iteration(0), - maxMomentum(maxMomentum) + momentum(momentum) { // Nothing to do. } @@ -78,29 +78,22 @@ class NesterovMomentumUpdate const double stepSize, const arma::mat& gradient) { - iteration++; + velocity = momentum * velocity - stepSize * gradient; - double momentumT = 0.5; - - velocity = momentumT * velocity - stepSize * gradient; - - iterate += velocity; + iterate += momentum * velocity - stepSize * gradient; } - //! Get the value used to initialise the maximum momentum coefficient. - double MaxMomentum() const { return maxMomentum; } - //! Modify the value used to initialise the maximum momentum coefficient. - double& MaxMomentum() { return maxMomentum; } + //! Get the value used to initialise the momentum coefficient. + double Momentum() const { return momentum; } + //! Modify the value used to initialise the momentum coefficient. + double& Momentum() { return momentum; } private: // The velocity matrix. arma::mat velocity; - // The number of iterations. - double iteration; - - // Maximum momentum coefficient - double maxMomentum; + // Momentum coefficient + double momentum; }; } // namespace optimization diff --git a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp index 442fa80ed7..e24e97148c 100644 --- a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp +++ b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp @@ -31,7 +31,7 @@ BOOST_AUTO_TEST_SUITE(NesterovMomentumSGDTest); BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) { SGDTestFunction f; - NesterovMomentumUpdate nesterovMomentumUpdate(0.999); + NesterovMomentumUpdate nesterovMomentumUpdate(0.7); NesterovMomentumSGD s(0.0003, 1, 2500000, 1e-9, true, nesterovMomentumUpdate); @@ -66,7 +66,7 @@ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) { // Create the generalized Rosenbrock function. GeneralizedRosenbrockFunction f(i); - NesterovMomentumUpdate nesterovMomentumUpdate(0.999); + NesterovMomentumUpdate nesterovMomentumUpdate(0.4); NesterovMomentumSGD s(0.0008, 1, 0, 1e-15, true, nesterovMomentumUpdate); arma::mat coordinates = f.GetInitialPoint(); From 765c1be1572bca2052934198cc4568020af4b4e1 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Sun, 11 Mar 2018 20:25:42 +0530 Subject: [PATCH 36/79] Updated tests --- src/mlpack/tests/nesterov_momentum_sgd_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp index e24e97148c..50feec5b64 100644 --- a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp +++ b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp @@ -31,7 +31,7 @@ BOOST_AUTO_TEST_SUITE(NesterovMomentumSGDTest); BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) { SGDTestFunction f; - NesterovMomentumUpdate nesterovMomentumUpdate(0.7); + NesterovMomentumUpdate nesterovMomentumUpdate(0.9); NesterovMomentumSGD s(0.0003, 1, 2500000, 1e-9, true, nesterovMomentumUpdate); @@ -66,8 +66,8 @@ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) { // Create the generalized Rosenbrock function. GeneralizedRosenbrockFunction f(i); - NesterovMomentumUpdate nesterovMomentumUpdate(0.4); - NesterovMomentumSGD s(0.0008, 1, 0, 1e-15, true, nesterovMomentumUpdate); + NesterovMomentumUpdate nesterovMomentumUpdate(0.9); + NesterovMomentumSGD s(0.0001, 1, 0, 1e-15, true, nesterovMomentumUpdate); arma::mat coordinates = f.GetInitialPoint(); double result = s.Optimize(f, coordinates); From e81fb5c8a5bea80ab1c659b4900131cbcacd1d29 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Sun, 11 Mar 2018 20:33:18 +0530 Subject: [PATCH 37/79] Remove unused initialization --- .../optimizers/sgd/update_policies/nesterov_momentum_update.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 5616fd68cd..ab4c66320d 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -44,7 +44,6 @@ class NesterovMomentumUpdate * */ NesterovMomentumUpdate(const double momentum = 0.5) : - iteration(0), momentum(momentum) { // Nothing to do. From e3e1e9b4d578b2531010259143b3cef635c56279 Mon Sep 17 00:00:00 2001 From: Moksh Jain Date: Sun, 11 Mar 2018 23:43:24 +0530 Subject: [PATCH 38/79] update contributors list --- COPYRIGHT.txt | 1 + src/mlpack/core.hpp | 1 + 2 files changed, 2 insertions(+) diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index d6fe3c0185..9ed8b885f5 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -95,6 +95,7 @@ Copyright: Copyright 2018, Wenhao Huang Copyright 2018, Roberto Hueso Copyright 2018, Prabhat Sharma + Copyright 2018, Moksh Jain License: BSD-3-clause All rights reserved. diff --git a/src/mlpack/core.hpp b/src/mlpack/core.hpp index 3a44877ced..b460a97fa5 100644 --- a/src/mlpack/core.hpp +++ b/src/mlpack/core.hpp @@ -238,6 +238,7 @@ * - Wenhao Huang * - Roberto Hueso * - Prabhat Sharma + * - Moksh Jain */ // First, include all of the prerequisites. From 79f8a9d549986128d056bddebcc8809d8d017c2c Mon Sep 17 00:00:00 2001 From: Moksh Jain Date: Sun, 11 Mar 2018 23:50:50 +0530 Subject: [PATCH 39/79] update contributors list --- COPYRIGHT.txt | 1 + src/mlpack/core.hpp | 1 + 2 files changed, 2 insertions(+) diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index 609b873fb0..2cdaa62c00 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -96,6 +96,7 @@ Copyright: Copyright 2018, Roberto Hueso Copyright 2018, Prabhat Sharma Copyright 2018, Tan Jun An + Copyright 2018, Moksh Jain License: BSD-3-clause All rights reserved. diff --git a/src/mlpack/core.hpp b/src/mlpack/core.hpp index 845ccc723a..d9a8b636ca 100644 --- a/src/mlpack/core.hpp +++ b/src/mlpack/core.hpp @@ -239,6 +239,7 @@ * - Roberto Hueso * - Prabhat Sharma * - Tan Jun An + * - Moksh Jain */ // First, include all of the prerequisites. From 7668ea4ea75e25e8ac465ff0816456a17387d5ed Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Mon, 12 Mar 2018 00:31:46 +0530 Subject: [PATCH 40/79] Fix main_tests --- .../tests/main_tests/decision_tree_test.cpp | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/mlpack/tests/main_tests/decision_tree_test.cpp b/src/mlpack/tests/main_tests/decision_tree_test.cpp index 1b8403a8fe..56ae2c88de 100644 --- a/src/mlpack/tests/main_tests/decision_tree_test.cpp +++ b/src/mlpack/tests/main_tests/decision_tree_test.cpp @@ -201,36 +201,40 @@ BOOST_AUTO_TEST_CASE(DecisionMinimumGainSplitTest) */ BOOST_AUTO_TEST_CASE(DecisionRegularisationTest) { - // Completely random dataset with no structure. - arma::mat dataset(10, 1000, arma::fill::randu); - arma::Row labels(1000); - for (size_t i = 0; i < 1000; ++i) - labels[i] = i % 3; // 3 classes. - arma::rowvec weights(labels.n_elem); - weights.ones(); + arma::mat inputData; + DatasetInfo info; + if (!data::Load("braziltourism.arff", inputData, info)) + BOOST_FAIL("Cannot load train dataset braziltourism.arff!"); + + arma::Row labels; + if (!data::Load("braziltourism_labels.txt", labels)) + BOOST_FAIL("Cannot load labels for braziltourism_labels.txt"); + + // Initialize an all-ones weight matrix. + arma::mat weights(1, labels.n_cols, arma::fill::ones); // Input training data. - SetInputParam("training", dataset); + SetInputParam("training", std::make_tuple(info, inputData)); SetInputParam("labels", labels); SetInputParam("weights", weights); SetInputParam("minimum_gain_split", 1e-7); // Input test data. - SetInputParam("test", dataset); + SetInputParam("test", std::make_tuple(info, inputData)); arma::Row pred; mlpackMain(); pred = std::move(CLI::GetParam>("predictions")); // Input training data. - SetInputParam("training", dataset); + SetInputParam("training", std::make_tuple(info, inputData)); SetInputParam("labels", std::move(labels)); SetInputParam("weights", std::move(weights)); SetInputParam("minimum_gain_split", 0.01); // Input test data. - SetInputParam("test", std::move(dataset)); + SetInputParam("test", std::move(std::make_tuple(info, inputData))); arma::Row predRegularised; mlpackMain(); predRegularised = std::move(CLI::GetParam>("predictions")); From b40cc23aa63a143d8168cb1c46569e29922478a8 Mon Sep 17 00:00:00 2001 From: Moksh Jain Date: Mon, 12 Mar 2018 18:08:23 +0530 Subject: [PATCH 41/79] fix typos --- .../optimizers/adam/optimisticadam_update.hpp | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/mlpack/core/optimizers/adam/optimisticadam_update.hpp b/src/mlpack/core/optimizers/adam/optimisticadam_update.hpp index 27e0dd8484..26d4162c7d 100644 --- a/src/mlpack/core/optimizers/adam/optimisticadam_update.hpp +++ b/src/mlpack/core/optimizers/adam/optimisticadam_update.hpp @@ -2,8 +2,8 @@ * @file optimisticadam_update.hpp * @author Moksh Jain * - * OptmisticAdam optimizer. Optimistic Adam is an an algorithm which uses - * Optimistic Mirror Descent with the Adam optimizer. + * OptmisticAdam optimizer. Implements Optimistic Adam, an algorithm which + * uses Optimistic Mirror Descent with the Adam optimizer. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -19,8 +19,11 @@ namespace mlpack { namespace optimization { /** - * OptimisticAdam is an optimizer which uses Optmistic Mirror Descent with - * the Adam Optimizer. + * OptimisticAdam is an optimizer which implements the Optimistic Adam + * algorithm which uses Optmistic Mirror Descent with the Adam Optimizer. + * It addresses the problem of limit cycling while training GANs. It uses + * OMD to achieve faster regret rates in solving the zero sum game of + * training a GAN. * * For more information, see the following. * @@ -30,7 +33,7 @@ namespace optimization { * Haoyang Zeng}, * title = {Training GANs with Optimism}, * year = {2017}, - * url = {https://openreview.net/forum?id=SJJySbbAZ} + * url = {https://arxiv.org/abs/1711.00141} * } * @endcode */ @@ -40,7 +43,7 @@ class OptimisticAdamUpdate /** * Construct the OptimisticAdam update policy with the given parameters. * - * @param epsilon The epsilon value used to initialise the squared gradient + * @param epsilon The epsilon value used to initialize the squared gradient * parameter. * @param beta1 The smoothing parameter. * @param beta2 The second moment coefficient. @@ -101,9 +104,9 @@ class OptimisticAdamUpdate g = std::move(update); } - //! Get the value used to initialise the squared gradient parameter. + //! Get the value used to initialize the squared gradient parameter. double Epsilon() const { return epsilon; } - //! Modify the value used to initialise the squared gradient parameter. + //! Modify the value used to initialize the squared gradient parameter. double& Epsilon() { return epsilon; } //! Get the smoothing parameter. @@ -117,7 +120,7 @@ class OptimisticAdamUpdate double& Beta2() { return beta2; } private: - // The epsilon value used to initialise the squared gradient parameter. + // The epsilon value used to initialize the squared gradient parameter. double epsilon; // The smoothing parameter. @@ -131,7 +134,7 @@ class OptimisticAdamUpdate // The exponential moving average of squared gradient values. arma::mat v; - // The previous update + // The previous update. arma::mat g; // The number of iterations. From c24cdc94a0b2421d38c30cb31aa0c714a5daf9d9 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 12 Mar 2018 17:33:14 +0100 Subject: [PATCH 42/79] Add basic ann tutorial. --- doc/tutorials/ann/ann.txt | 673 ++++++++++++++++++++++++++++++++++++ doc/tutorials/tutorials.txt | 1 + 2 files changed, 674 insertions(+) create mode 100644 doc/tutorials/ann/ann.txt diff --git a/doc/tutorials/ann/ann.txt b/doc/tutorials/ann/ann.txt new file mode 100644 index 0000000000..c41d6ea626 --- /dev/null +++ b/doc/tutorials/ann/ann.txt @@ -0,0 +1,673 @@ +/*! +@file ann.txt +@author Marcus Edel (kurg.org) +@brief Tutorial for how to use the neural network code in mlpack. + +@page anntutorial Neural Network tutorial + +@section intro_anntut Introduction + +There is vast literature on neural networks and their uses, as well as +strategies for choosing initial points effectively, keeping the algorithm from +converging in local minima, choosing the best model structure, choosing the best +optimizers, and so forth. mlpack implements many of these building blocks, +making it very easy to create different neural networks in a modular way. + +mlpack currently implements two easy-to-use forms of neural networks: \c Feed- +Forward \c Networks (this includes convolutional neural networks) and \c +Recurrent \c Neural \c Networks. + +@section toc_anntut Table of Contents + +This tutorial is split into the following sections: + + - \ref intro_anntut + - \ref toc_anntut + - \ref model_api_anntut + - \ref layer_api_anntut + - \ref model_setup_training_anntut + - \ref model_saving_loading_anntut + - \ref extracting_parameters_anntut + - \ref further_anntut + +@section model_api_anntut Model API + +There are two main neural network classes that are meant to be used as container +for neural network layers that \b mlpack implements; each class is suited to a +different setting: + +- \c FFN: the Feed Forward Network model provides a means to plug layers + together in a feed-forward fully connected manner. This is the 'standard' + type of deep learning model, and includes convolutional neural networks + (CNNs). + +- \c RNN: the Recurrent Neural Network model provides a means to consider + successive calls to forward as different time-steps in a sequence. This is + often used for time sequence modeling tasks, such as predicting the next + character in a sequence. + +Below is some basic guidance on what should be used. Note that the question of +"which algorithm should be used" is a very difficult question to answer, so the +guidance below is just that---guidance---and may not be right for a particular +problem. + + - \c Feed-forward Networks allow signals or inputs to travel one way only. + There is no feedback within the network; for instance, the output of any + layer does only affect the upcoming layer. That makes Feed-Forward Networks + straightforward and very effective. They are extensively used in pattern + recognition and are ideally suitable for modeling relationships between a + set of input and one or more output variables. + + + - \c Recurrent Networks allow signals or inputs to travel in both directions by + introducing loops in the network. Computations derived from earlier inputs are + fed back into the network, which gives the recurrent network some kind of + memory. RNNs are currently being used for all kinds of sequential tasks; for + instance, time series prediction, sequence labeling, and + sequence classification. + +In order to facilitate consistent implementations, the \c FFN and \c RNN classes +have a number of methods in common: + + - \c Train(): trains the initialized model on the given input data. Optionally + an optimizer object can be passed to control the optimization process. + + - \c Predict(): predicts the responses to a given set of predictors. Note the + responses will reflect the output of the specified output layer. + + - \c Add(): this method can be used to add a layer to the model. + +@note +To be able to optimize the network, both classes implement the OptimizerFunction +API; see \ref optimizertutorial "Optimizer API" for more information. In short, +the \c FNN and \c RNN class implement two methods: \c Evaluate() and \c +Gradient(). This enables the optimization given some learner and some +performance measure. + +Similar to the existing layer infrastructure, the \c FFN and \c RNN classes are +very extensible, having the following template arguments; which can be modified +to change the behavior of the network: + + - \c OutputLayerType: this type defines the output layer used to evaluate the + network; by default, \c NegativeLogLikelihood is used. + + - \c InitializationRuleType: this type defines the method by which initial + parameters are set; by default, \c RandomInitialization is used. + +@code +template< + typename OutputLayerType = NegativeLogLikelihood<>, + typename InitializationRuleType = RandomInitialization +> +class FNN; +@endcode + +Internally, the \c FFN and \c RNN class keeps an instantiated \c OutputLayerType +class (which can be given in the constructor). This is useful for using +different loss functions like the Negative-Log-Likelihood function or the \c +VRClassReward function, which takes an optional score parameter. Therefore, you +can write a non-static OutputLayerType class and use it seamlessly in +combination with the \c FNN and \c RNN class. The same applies to the \c +InitializationRuleType template parameter. + +By choosing different components for each of these template classes in +conjunction with the \c Add() method, a very arbitrary network object can be +constructed. + +Below are several examples of how the \c FNN and \c RNN classes might be used. +The first examples focus on the \c FNN class, and the last shows how the \c +RNN class can be used. + +The simplest way to use the FNN<> class is to pass in a dataset with the +corresponding labels, and receive the classification in return. Note that the +dataset must be column-major – that is, one column corresponds to one point. See +the \ref matrices "matrices guide" for more information. + +The code below builds a simple feed-forward network with the default options, +then queries for the assignments for every point in the \c queries matrix. + +\dot +digraph G { + fontname = "Hilda 10" + rankdir=LR + splines=line + nodesep=.08; + ranksep=1; + edge [color=black, arrowsize=.5]; + node [fixedsize=true,label="",style=filled,color=none,fillcolor=gray,shape=circle] + + subgraph cluster_0 { + color=none; + node [style=filled, color=white, penwidth=15,fillcolor=black shape=circle]; + l10 l11 l12 l13 l14 l15 ; + label = Input; + } + + subgraph cluster_1 { + color=none; + node [style=filled, color=white, penwidth=15,fillcolor=gray shape=circle]; + l20 l21 l22 l23 l24 l25 l26 l27 ; + label = Linear; + } + + subgraph cluster_2 { + color=none; + node [style=filled, color=white, penwidth=15,fillcolor=gray shape=circle]; + l30 l31 l32 l33 l34 l35 l36 l37 ; + label = Linear; + } + + subgraph cluster_3 { + color=none; + node [style=filled, color=white, penwidth=15,fillcolor=black shape=circle]; + l40 l41 l42 ; + label = LogSoftMax; + } + + l10 -> l20 l10 -> l21 l10 -> l22 l10 -> l23 l10 -> l24 l10 -> l25 + l10 -> l26 l10 -> l27 l11 -> l20 l11 -> l21 l11 -> l22 l11 -> l23 + l11 -> l24 l11 -> l25 l11 -> l26 l11 -> l27 l12 -> l20 l12 -> l21 + l12 -> l22 l12 -> l23 l12 -> l24 l12 -> l25 l12 -> l26 l12 -> l27 + l13 -> l20 l13 -> l21 l13 -> l22 l13 -> l23 l13 -> l24 l13 -> l25 + l13 -> l26 l13 -> l27 l14 -> l20 l14 -> l21 l14 -> l22 l14 -> l23 + l14 -> l24 l14 -> l25 l14 -> l26 l14 -> l27 l15 -> l20 l15 -> l21 + l15 -> l22 l15 -> l23 l15 -> l24 l15 -> l25 l15 -> l26 l15 -> l27 + l20 -> l30 l20 -> l31 l20 -> l32 l20 -> l33 l20 -> l34 l20 -> l35 + l20 -> l36 l20 -> l37 l21 -> l30 l21 -> l31 l21 -> l32 l21 -> l33 + l21 -> l34 l21 -> l35 l21 -> l36 l21 -> l37 l22 -> l30 l22 -> l31 + l22 -> l32 l22 -> l33 l22 -> l34 l22 -> l35 l22 -> l36 l22 -> l37 + l23 -> l30 l23 -> l31 l23 -> l32 l23 -> l33 l23 -> l34 l23 -> l35 + l23 -> l36 l23 -> l37 l24 -> l30 l24 -> l31 l24 -> l32 l24 -> l33 + l24 -> l34 l24 -> l35 l24 -> l36 l24 -> l37 l25 -> l30 l25 -> l31 + l25 -> l32 l25 -> l33 l25 -> l34 l25 -> l35 l25 -> l36 l25 -> l37 + l26 -> l30 l26 -> l31 l26 -> l32 l26 -> l33 l26 -> l34 l26 -> l35 + l26 -> l36 l26 -> l37 l27 -> l30 l27 -> l31 l27 -> l32 l27 -> l33 + l27 -> l34 l27 -> l35 l27 -> l36 l27 -> l37 l30 -> l40 l30 -> l41 + l30 -> l42 l31 -> l40 l31 -> l41 l31 -> l42 l32 -> l40 l32 -> l41 + l32 -> l42 l33 -> l40 l33 -> l41 l33 -> l42 l34 -> l40 l34 -> l41 + l34 -> l42 l35 -> l40 l35 -> l41 l35 -> l42 l36 -> l40 l36 -> l41 + l36 -> l42 l37 -> l40 l37 -> l41 l37 -> l42 +} +\enddot +@note +The number of inputs in the above graph doesn't match with the real +number of features in the thyroid dataset and are just used as an abstract +representation. + +@code +// Load the training set. +arma::mat dataset; +data::Load("thyroid_train.csv", dataset, true); + +// Split the labels from the training set. +arma::mat trainData = dataset.submat(0, 0, dataset.n_rows - 4, + dataset.n_cols - 1); + +// Split the data from the training set. +arma::mat trainLabelsTemp = dataset.submat(dataset.n_rows - 3, 0, + dataset.n_rows - 1, dataset.n_cols - 1); + +// Initialize the network. +FFN<> model; +model.Add >(trainData.n_rows, 8); +model.Add >(); +model.Add >(8, 3); +model.Add >(); + +// Train the model. +model.Train(trainData, trainLabels); + +// Use the Predict method to get the assignments. +arma::mat assignments; +model.Predict(trainData, assignments); +@endcode + +Now, the matrix assignments holds the classification of each point in the +dataset. + +In the next example, we create simple noisy sine sequences, which are trained +later on, using the RNN class. + +@code +void GenerateNoisySines(arma::mat& data, + arma::mat& labels, + const size_t points, + const size_t sequences, + const double noise = 0.3) +{ + arma::colvec x = arma::linspace>(0, + points - 1, points) / points * 20.0; + arma::colvec y1 = arma::sin(x + arma::as_scalar(arma::randu(1)) * 3.0); + arma::colvec y2 = arma::sin(x / 2.0 + arma::as_scalar(arma::randu(1)) * 3.0); + + data = arma::zeros(points, sequences * 2); + labels = arma::zeros(2, sequences * 2); + + for (size_t seq = 0; seq < sequences; seq++) + { + data.col(seq) = arma::randu(points) * noise + y1 + + arma::as_scalar(arma::randu(1) - 0.5) * noise; + labels(0, seq) = 1; + + data.col(sequences + seq) = arma::randu(points) * noise + y2 + + arma::as_scalar(arma::randu(1) - 0.5) * noise; + labels(1, sequences + seq) = 1; + } + + const size_t rho = 10; + + // Generate 12 (2 * 6) noisy sines. A single sine contains rho + // points/features. + arma::mat input, labelsTemp; + GenerateNoisySines(input, labelsTemp, rho, 6); + + arma::mat labels = arma::zeros(rho, labelsTemp.n_cols); + for (size_t i = 0; i < labelsTemp.n_cols; ++i) + { + const int value = arma::as_scalar(arma::find( + arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; + labels.col(i).fill(value); + } + + /** + * Construct a network with 1 input unit, 4 hidden units and 10 output + * units. The hidden layer is connected to itself. The network structure + * looks like: + * + * Input Hidden Output + * Layer(1) Layer(4) Layer(10) + * +-----+ +-----+ +-----+ + * | | | | | | + * | +------>| +------>| | + * | | ..>| | | | + * +-----+ . +--+--+ +-----+ + * . . + * . . + * ....... + */ + Add<> add(4); + Linear<> lookup(1, 4); + SigmoidLayer<> sigmoidLayer; + Linear<> linear(4, 4); + Recurrent<> recurrent(add, lookup, linear, sigmoidLayer, rho); + + RNN<> model(rho); + model.Add >(); + model.Add(recurrent); + model.Add >(4, 10); + model.Add >(); + + StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch */, -100); + model.Train(input, labels, opt); +} +@endcode + +For further examples on the ann usage of the ann classes, see [mlpack +models](https://github.com/mlpack/models) + +@section layer_api_anntut Layer API + +In order to facilitate consistent implementations, we have defined a LayerType +API that describes all the methods that a Layer may implement. mlpack offers a +few variations of this API, each designed to cover some of the model +characteristics mentioned in the previous section. Any Layer requires the +implementation of a \c Forward() method. The interface looks like: + +@code +template +void Forward(const arma::Mat&& input, arma::Mat&& output); +@endcode + +The method should calculate the output of the layer given the input matrix and +store the result in the given output matrix. Next, any Layer must implement the +Backward() method, which uses certain computations obtained during forward and +should calculate the function f(x) by propagating x backward trough f: + +@code +template +void Backward(const arma::Mat&& input, + arma::Mat&& gy, + arma::Mat&& g); +@endcode + +Finally, if the layer is differentiable, the layer must also implement +a Gradient() method: + +@code +template +void Gradient(const arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& gradient); +@endcode + +The Gradient function should calculate the gradient with respect to the input +activations \c input and calculated errors \c error and place the results into +the gradient matrix object \c gradient that is passed as an argument. + +@note +Note that each method accepts a template parameter InputType, OutputType +or GradientType, which may be arma::mat (dense Armadillo matrix) or arma::sp_mat +(sparse Armadillo matrix). This allows support for both sparse-supporting and +non-sparse-supporting Layer without explicitly passing the type. + +In addition, each layer must implement the Parameters(), InputParameter(), +OutputParameter(), Delta() methods, differentiable layer should also provide +access to the gradient by implementing the Gradient(), Parameters() member +function. Note each function is a single line that looks like: + +@code +OutputDataType const& Parameters() const { return weights; } +@endcode + +Below is an example that shows each function with some additional boilerplate +code. + +@note +Note this is not an actual layer but instead an example that exists to +show and document all the functions that mlpack layer must implement. For a +better overview of the various layers, see \ref mlpack::ann. Also be aware that the +implementations of each of the methods in this example are entirely fake and do +not work; this example exists for its API, not its implementation. + +Note that layer sometimes have different properties. These properties are +known at compile-time through the mlpack::ann::LayerTraits class, and some +properties may imply the existence (or non-existence) of certain functions. +Refer to the LayerTraits @ref LayerTraits for more documentation on that. + +The two template parameters below must be template parameters to the layer, in +the order given below. More template parameters are fine, but they must come +after the first two. + + - \c InputDataType: this defines the internally used input type for example to + store the parameter matrix. Note, a layer could be built on a dense matrix or + a sparse matrix. All mlpack trees should be able to support any Armadillo- + compatible matrix type. When the layer is written it should be assumed that + MatType has the same functionality as arma::mat. Note that + + - \c OutputDataType: this defines the internally used input type for example to + store the parameter matrix. Note, a layer could be built on a dense matrix or + a sparse matrix. All mlpack trees should be able to support any Armadillo- + compatible matrix type. When the layer is written it should be assumed that + MatType has the same functionality as arma::mat. + +@code +template +class ExampleLayer +{ + public: + ExampleLayer(const size_t inSize, const size_t outSize) : + inputSize(inSize), outputSize(outSize) + { + /* Nothing to do here */ + } +} +@endcode + +The constructor for \c ExampleLayer will build the layer given the input and +output size. Note that, if the input or output size information isn't used +internally it's not necessary to provide a specific constructor. Also, one could +add additional or other information that are necessary for the layer +construction. One example could be: + +@code +ExampleLayer(const double ratio = 0.5) : ratio(ratio) {/* Nothing to do here*/} +@endcode + +When this constructor is finished, the entire layer will be built and is ready +to be used. Next, as pointed out above, each layer has to follow the LayerType +API, so we must implement some additional functions. + +@code +template +void Forward(const InputType&& input, OutputType&& output) +{ + output = arma::ones(input.n_rows, input.n_cols); +} + +template +void Backward(const InputType&& input, ErrorType&& gy, GradientType&& g) +{ + g = arma::zeros(gy.n_rows, gy.n_cols) + gy; +} + +template +void Gradient(const InputType&& input, + ErrorType&& error, + GradientType&& gradient) +{ + gradient = arma::zeros(input.n_rows, input.n_cols) * error; +} +@endcode + +The three functions \c Forward(), \c Backward() and \c Gradient() (which is +needed for a differentiable layer) contain the main logic of the layer. The +following functions are just to access and manipulate the different layer +parameter. + +@code +OutputDataType& Parameters() { return weights; } +InputDataType& InputParameter() { return inputParameter; } +OutputDataType& OutputParameter() { return outputParameter; } +OutputDataType& Delta() { return delta; } +OutputDataType& Gradient() { return gradient; } +@endcode + +Since some of this methods return internal class members we have to define them. + +@code +private: + size_t inSize, outSize; + OutputDataType weights, delta, gradient, outputParameter; + InputDataType inputParameter; +@endcode + +Note some members are just here so \c ExampleLayer compiles without warning. +For instance, \c inputSize is not required to be a member of every type of +layer. + +There is one last method that is especially interesting for a layer that shares +parameter. Since the layer weights are set once the complete model is defined, +it's not possible to split the weights during the construction time. To solve +this issue, a layer can implement the \c Reset() method which is called once the +layer parameter is set. + +@section model_setup_training_anntut Model Setup & Training + +Once the base container is selected (\c FNN or \c RNN), the \c Add method can be +used to add layers to the model. The code below adds two linear layers to the +model---the first takes 512 units as input and gives 256 output units, and +the second takes 256 units as input and gives 128 output units. + +@code +FFN<> model; +model.Add >(512, 256); +model.Add >(256, 128); +@endcode + +The model is trained on Armadillo matrices. For training a model, you will +typically use the \c Train() function: + +@code +arma::mat trainingSet, trainingLabels; +model.Train(trainingSet, trainingLabels); +@endcode + +You can use mlpack's \c Load() function to load a dataset like this: + +@code +arma::mat trainingSet; +data::Load("dataset.csv", dataset, true); +@endcode + +@code +$ cat dataset.csv +0, 1, 4 +1, 0, 5 +1, 1, 1 +2, 0, 2 +@endcode + +The type does not necessarily need to be a CSV; it can be any supported storage +format, assuming that it is a coordinate-format file in the format specified +above. For more information on mlpack file formats, see the documentation for +mlpack::data::Load(). + +@note +It’s often a good idea to normalize or standardize your data, for example using: + +@code +for (size_t i = 0; i < dataset.n_cols; ++i) + dataset.col(i) /= norm(dataset.col(i), 2); +@endcode + +Also, it is possible to retrain a model with new parameters or with +a new reference set. This is functionally equivalent to creating a new model. + +@section model_saving_loading_anntut Saving & Loading + +Using \c boost::serialization (for more information about the internals see +[Serialization - Boost C++ Libraries](www.boost.org/libs/serialization/doc/)), +mlpack is able to load and save machine learning models with ease. To save a +trained neural network to disk. The example below builds a model on the \c +thyroid dataset and then saves the model to the file \c model.xml for later use. + +@code +// Load the training set. +arma::mat dataset; +data::Load("thyroid_train.csv", dataset, true); + +// Split the labels from the training set. +arma::mat trainData = dataset.submat(0, 0, dataset.n_rows - 4, + dataset.n_cols - 1); + +// Split the data from the training set. +arma::mat trainLabelsTemp = dataset.submat(dataset.n_rows - 3, 0, + dataset.n_rows - 1, dataset.n_cols - 1); + +// Initialize the network. +FFN<> model; +model.Add >(trainData.n_rows, 3); +model.Add >(); +model.Add >(); + +// Train the model. +model.Train(trainData, trainLabels); + +// Use the Predict method to get the assignments. +arma::mat assignments; +model.Predict(trainData, assignments); + +data::Save("model.xml", "model", model, false); +@endcode + +After this, the file model.xml will be available in the current working +directory. + +Now, we can look at the output model file, \c model.xml: + +@code +$ cat model.xml + + + + + + 66 + 1 + 66 + 0 + -7.55971528334903642e+00 + -9.95435955058058930e+00 + 9.31133928948225353e+00 + -5.36784434861701953e+00 + ... + + 0 + 0 + + 0 + 0 + 0 + 0 + + + 3 + 0 + + 18 + + 21 + 3 + + + + 2 + + + + 20 + + + + + +@endcode + +As you can see, the \c section of \c model.xml contains the trained +network weights. We can see that this section also contains the network input +size, which is 66 rows and 1 column. Note that in this example, we used three +different layers, as can be seen by looking at the \c section. Each +node has a unique id that is used to reconstruct the model when loading. + +The models can also be saved as \c .bin or \c .txt; the \c .xml format provides +a human-inspectable format (though the models tend to be quite complex and may +be difficult to read). These models can then be re-used to be used for +classification or other tasks. + +So, instead of saving or training a network, mlpack can also load a pre-trained +model. For instance, the example below will load the model from \c model.xml and +then generate the class predictions for the \c thyroid test dataset. + +@code +data::Load("thyroid_test.csv", dataset, true); + +arma::mat testData = dataset.submat(0, 0, dataset.n_rows - 4, + dataset.n_cols - 1); + +data::Load("model.xml", "model", model); + +arma::mat predictions; +model.Predict(testData, predictions); +@endcode + +This enables the possibility to distribute a model without having to train it +first or simply to save a model for later use. Note that loading will also work +on different machines. + +@section extracting_parameters_anntut Extracting Parameters + +To access the weights from the neural network layers, you can call the following +function on any initialized network: + +@code +model.Parameters(); +@endcode + +which will return the complete model parameters as an armadillo matrix object; +however often it is useful to not only have the parameters for the complete +network, but the parameters of a specific layer. Another method, \c Model(), +makes this easily possible: + +@code +model.Model()[1].Parameters(); +@endcode + +In the example above, we get the weights of the second layer. + +@section further_anntut Further documentation + +For further documentation on the ann classes, consult the \ref mlpack::ann +"complete API documentation". + +*/ diff --git a/doc/tutorials/tutorials.txt b/doc/tutorials/tutorials.txt index 169902e54d..a8df927130 100644 --- a/doc/tutorials/tutorials.txt +++ b/doc/tutorials/tutorials.txt @@ -39,6 +39,7 @@ progress to complex, extensible uses. - \ref cftutorial - \ref akfntutorial - \ref cnetutorial + - \ref anntutorial @section policy_tut Policy Class Documentation From 3f87ab1d419493dead8ef59250c02cc7aacc0adb Mon Sep 17 00:00:00 2001 From: Moksh Jain Date: Mon, 12 Mar 2018 22:11:05 +0530 Subject: [PATCH 43/79] added description of optimisticadam --- src/mlpack/core/optimizers/adam/optimisticadam_update.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mlpack/core/optimizers/adam/optimisticadam_update.hpp b/src/mlpack/core/optimizers/adam/optimisticadam_update.hpp index 26d4162c7d..06b45d9274 100644 --- a/src/mlpack/core/optimizers/adam/optimisticadam_update.hpp +++ b/src/mlpack/core/optimizers/adam/optimisticadam_update.hpp @@ -23,7 +23,8 @@ namespace optimization { * algorithm which uses Optmistic Mirror Descent with the Adam Optimizer. * It addresses the problem of limit cycling while training GANs. It uses * OMD to achieve faster regret rates in solving the zero sum game of - * training a GAN. + * training a GAN. It consistently achieves a smaller KL divergnce with + * respect to the true underlying data distribution. * * For more information, see the following. * From 60b6cd1e489c1cbaa30fbe990eb7c9f732593ada Mon Sep 17 00:00:00 2001 From: Wenhao Huang Date: Tue, 13 Mar 2018 23:16:52 +0800 Subject: [PATCH 44/79] another idea --- src/mlpack/core/math/random.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/math/random.hpp b/src/mlpack/core/math/random.hpp index c30c011660..85870b280c 100644 --- a/src/mlpack/core/math/random.hpp +++ b/src/mlpack/core/math/random.hpp @@ -54,7 +54,7 @@ inline void RandomSeed(const size_t seed) #if (BINDING_TYPE == BINDING_TYPE_TEST) inline void SetFixedRandomSeed() { - const size_t seed = 54321; + const static size_t seed = rand(); randGen.seed((uint32_t) seed); srand((unsigned int) seed); arma::arma_rng::set_seed(seed); From 78a7032b72d63d7cc5b34630b220a0dd1a555cf5 Mon Sep 17 00:00:00 2001 From: Wenhao Huang Date: Tue, 13 Mar 2018 23:18:01 +0800 Subject: [PATCH 45/79] change comment --- src/mlpack/core/math/random.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/math/random.hpp b/src/mlpack/core/math/random.hpp index 85870b280c..8e90d9bcb8 100644 --- a/src/mlpack/core/math/random.hpp +++ b/src/mlpack/core/math/random.hpp @@ -49,7 +49,7 @@ inline void RandomSeed(const size_t seed) } /** - * Set the random seed to a predefined seed. + * Set the random seed to a fixed number. */ #if (BINDING_TYPE == BINDING_TYPE_TEST) inline void SetFixedRandomSeed() From 4d3f3b66e0aa9085f17d7487dae4f2d7c33eb634 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 13 Mar 2018 22:41:26 +0100 Subject: [PATCH 46/79] Grammar and punctuation fix. --- doc/tutorials/ann/ann.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/tutorials/ann/ann.txt b/doc/tutorials/ann/ann.txt index c41d6ea626..866da19758 100644 --- a/doc/tutorials/ann/ann.txt +++ b/doc/tutorials/ann/ann.txt @@ -302,8 +302,8 @@ void GenerateNoisySines(arma::mat& data, } @endcode -For further examples on the ann usage of the ann classes, see [mlpack -models](https://github.com/mlpack/models) +For further examples on the usage of the ann classes, see [mlpack +models](https://github.com/mlpack/models). @section layer_api_anntut Layer API From d0b6f69d9e71b3b3f812c249429fc7e69e8982c5 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Wed, 14 Mar 2018 17:27:58 +0530 Subject: [PATCH 47/79] Minor style fixes --- src/mlpack/core/optimizers/sgd/sgd.hpp | 2 +- .../nesterov_momentum_update.hpp | 3 +-- .../tests/nesterov_momentum_sgd_test.cpp | 21 ++++++------------- 3 files changed, 8 insertions(+), 18 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/sgd.hpp b/src/mlpack/core/optimizers/sgd/sgd.hpp index 047e871048..523539ec73 100644 --- a/src/mlpack/core/optimizers/sgd/sgd.hpp +++ b/src/mlpack/core/optimizers/sgd/sgd.hpp @@ -107,7 +107,7 @@ class SGD * @param resetPolicy Flag that determines whether update policy parameters * are reset before every Optimize call. */ - SGD(const double stepSize = 0.001, + SGD(const double stepSize = 0.01, const size_t batchSize = 32, const size_t maxIterations = 100000, const double tolerance = 1e-5, diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index ab4c66320d..8bbede71c7 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -35,7 +35,6 @@ namespace optimization { * } * @endcode */ - class NesterovMomentumUpdate { public: @@ -91,7 +90,7 @@ class NesterovMomentumUpdate // The velocity matrix. arma::mat velocity; - // Momentum coefficient + // The Momentum coefficient. double momentum; }; diff --git a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp index 50feec5b64..265a112965 100644 --- a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp +++ b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp @@ -28,6 +28,9 @@ using namespace mlpack::optimization::test; BOOST_AUTO_TEST_SUITE(NesterovMomentumSGDTest); +/* +* Tests the Nesterov Momentum SGD update policy. +*/ BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) { SGDTestFunction f; @@ -42,23 +45,11 @@ BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) BOOST_REQUIRE_SMALL(coordinates[0], 1e-3); BOOST_REQUIRE_SMALL(coordinates[1], 1e-7); BOOST_REQUIRE_SMALL(coordinates[2], 1e-7); - - // Compare with SGD with vanilla update. - SGDTestFunction f1; - StandardSGD s1(0.0003, 1, 2500000, 1e-9, true); - - arma::mat coordinates1 = f.GetInitialPoint(); - double result1 = s1.Optimize(f1, coordinates1); - - // Result doesn't converge in 2500000 iterations. - BOOST_REQUIRE_GT(result1 + 1.0, 0.05); - BOOST_REQUIRE_GE(coordinates1[0], 1e-3); - BOOST_REQUIRE_SMALL(coordinates1[1], 1e-7); - BOOST_REQUIRE_SMALL(coordinates1[2], 1e-7); - - BOOST_REQUIRE_LE(result, result1); } +/* +* Tests the Nesterov Momentum SGD with Generalized Rosenbrock Test. +*/ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) { // Loop over several variants. From 0454300939889066e5fa992b9b85b53c948c8748 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Wed, 14 Mar 2018 17:42:21 +0100 Subject: [PATCH 48/79] Fix style and grammar issues. --- doc/tutorials/ann/ann.txt | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/doc/tutorials/ann/ann.txt b/doc/tutorials/ann/ann.txt index 866da19758..546e39de11 100644 --- a/doc/tutorials/ann/ann.txt +++ b/doc/tutorials/ann/ann.txt @@ -308,9 +308,9 @@ models](https://github.com/mlpack/models). @section layer_api_anntut Layer API In order to facilitate consistent implementations, we have defined a LayerType -API that describes all the methods that a Layer may implement. mlpack offers a -few variations of this API, each designed to cover some of the model -characteristics mentioned in the previous section. Any Layer requires the +API that describes all the methods that a \c layer may implement. mlpack offers +a few variations of this API, each designed to cover some of the model +characteristics mentioned in the previous section. Any \c layer requires the implementation of a \c Forward() method. The interface looks like: @code @@ -319,9 +319,10 @@ void Forward(const arma::Mat&& input, arma::Mat&& output); @endcode The method should calculate the output of the layer given the input matrix and -store the result in the given output matrix. Next, any Layer must implement the -Backward() method, which uses certain computations obtained during forward and -should calculate the function f(x) by propagating x backward trough f: +store the result in the given output matrix. Next, any \c layer must implement +the Backward() method, which uses certain computations obtained during the +forward pass and should calculate the function f(x) by propagating x backward +through f: @code template @@ -348,7 +349,7 @@ the gradient matrix object \c gradient that is passed as an argument. Note that each method accepts a template parameter InputType, OutputType or GradientType, which may be arma::mat (dense Armadillo matrix) or arma::sp_mat (sparse Armadillo matrix). This allows support for both sparse-supporting and -non-sparse-supporting Layer without explicitly passing the type. +non-sparse-supporting \c layer without explicitly passing the type. In addition, each layer must implement the Parameters(), InputParameter(), OutputParameter(), Delta() methods, differentiable layer should also provide @@ -363,9 +364,9 @@ Below is an example that shows each function with some additional boilerplate code. @note -Note this is not an actual layer but instead an example that exists to -show and document all the functions that mlpack layer must implement. For a -better overview of the various layers, see \ref mlpack::ann. Also be aware that the +Note this is not an actual layer but instead an example that exists to show and +document all the functions that mlpack layer must implement. For a better +overview of the various layers, see \ref mlpack::ann. Also be aware that the implementations of each of the methods in this example are entirely fake and do not work; this example exists for its API, not its implementation. @@ -443,7 +444,7 @@ void Gradient(const InputType&& input, The three functions \c Forward(), \c Backward() and \c Gradient() (which is needed for a differentiable layer) contain the main logic of the layer. The following functions are just to access and manipulate the different layer -parameter. +parameters. @code OutputDataType& Parameters() { return weights; } From 642d3516d81f22cd32183b2fb35e02a8bf753579 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Thu, 15 Mar 2018 21:49:54 +0530 Subject: [PATCH 49/79] Minor grammar fixes --- .../sgd/update_policies/nesterov_momentum_update.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 8bbede71c7..befb974913 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -59,14 +59,14 @@ class NesterovMomentumUpdate */ void Initialize(const size_t rows, const size_t cols) { - // Initialize am empty velocity matrix. + // Initialize an empty velocity matrix. velocity = arma::zeros(rows, cols); } /** * Update step for SGD. The momentum term makes the convergence faster on the - * way as momentum term increases for dimensions pointing in the same and - * reduces updates for dimensions whose gradients change directions. + * way as momentum term increases for dimensions pointing in the same direction + * and reduces updates for dimensions whose gradients change directions. * * @param iterate Parameters that minimize the function. * @param stepSize Step size to be used for the given iteration. @@ -81,9 +81,9 @@ class NesterovMomentumUpdate iterate += momentum * velocity - stepSize * gradient; } - //! Get the value used to initialise the momentum coefficient. + //! Get the value used to initialize the momentum coefficient. double Momentum() const { return momentum; } - //! Modify the value used to initialise the momentum coefficient. + //! Modify the value used to initialize the momentum coefficient. double& Momentum() { return momentum; } private: From f8533ac6655ceb153703f8bc75c808069f59a30b Mon Sep 17 00:00:00 2001 From: Praveen Ch Date: Thu, 9 Mar 2017 03:48:58 +1100 Subject: [PATCH 50/79] Implemented the BatchNorm Layer --- src/mlpack/methods/ann/layer/CMakeLists.txt | 2 + src/mlpack/methods/ann/layer/batchnorm.hpp | 116 ++++++++++++++++++ .../methods/ann/layer/batchnorm_impl.hpp | 96 +++++++++++++++ src/mlpack/methods/ann/layer/layer_types.hpp | 6 + 4 files changed, 220 insertions(+) create mode 100644 src/mlpack/methods/ann/layer/batchnorm.hpp create mode 100644 src/mlpack/methods/ann/layer/batchnorm_impl.hpp diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt index 57be256193..c749376196 100644 --- a/src/mlpack/methods/ann/layer/CMakeLists.txt +++ b/src/mlpack/methods/ann/layer/CMakeLists.txt @@ -8,6 +8,8 @@ set(SOURCES base_layer.hpp bilinear_interpolation.hpp bilinear_interpolation_impl.hpp + batchnorm.hpp + batchnorm_impl.hpp concat.hpp concat_impl.hpp concat_performance.hpp diff --git a/src/mlpack/methods/ann/layer/batchnorm.hpp b/src/mlpack/methods/ann/layer/batchnorm.hpp new file mode 100644 index 0000000000..be984c485a --- /dev/null +++ b/src/mlpack/methods/ann/layer/batchnorm.hpp @@ -0,0 +1,116 @@ +/** + * @file batchnorm.hpp + * @author Marcus Edel + * + * Definition of the Batch Normalisation layer class as proposed by Ioffe et.al + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_BATCHNORM_HPP +#define MLPACK_METHODS_ANN_LAYER_BATCHNORM_HPP + +#include + +#include "layer_types.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class BatchNorm +{ + public: + + BatchNorm(); + + BatchNorm(const size_t size); + + void Reset(); + + template + void Forward(const arma::Mat&& input, arma::Mat&& output); + + template + void Backward(const arma::Mat&& input, + arma::Mat&& gy, + arma::Mat&& g); + + template + void Gradient(const arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& gradient); + + //! Get the parameters. + OutputDataType const& Parameters() const { return weights; } + //! Modify the parameters. + OutputDataType& Parameters() { return weights; } + + //! Get the input parameter. + InputDataType const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } + + + template + void Serialize(Archive& ar, const unsigned int /* version */); + + private: + + OutputDataType gamma; + + OutputDataType beta; + + OutputDataType weights; + + double eps; + + bool deterministic; + + size_t size; + + OutputDataType mean; + + OutputDataType trainingMean; + + OutputDataType variance; + + OutputDataType trainingVariance; + + OutputDataType gradient; + + OutputDataType delta; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; + +} +} + +#include "batchnorm_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/layer/batchnorm_impl.hpp b/src/mlpack/methods/ann/layer/batchnorm_impl.hpp new file mode 100644 index 0000000000..94c6d9f93e --- /dev/null +++ b/src/mlpack/methods/ann/layer/batchnorm_impl.hpp @@ -0,0 +1,96 @@ +/** + * @file batchnorm_impl.hpp + * @author Praveen Ch + * + * Implementation of the Batch Normalization Layer. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_METHODS_ANN_LAYER_BATCHNORM_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_BATCHNORM_IMPL_HPP + + +#include "batchnorm.hpp" + +namespace mlpack { +namespace ann { + +template +BatchNorm::BatchNorm() +{ + // Nothing to do here. +} + +template +BatchNorm::BatchNorm( + const size_t size) : size(size) +{ + weights.set_size(size + size, 1); +} + + +template +void BatchNorm::Reset() +{ + // variance = arma::mat(variance.memptr(), size, 1, false, false); + // mean = arma::mat(mean.memptr(), size, 1, false, false); + gamma = arma::mat(weights.memptr(), size, 1, false, false); + beta = arma::mat(weights.memptr() + gamma.n_elem, size, 1, false, false); +} + +template +template +void BatchNorm::Forward( + const arma::Mat&& input, arma::Mat&& output) +{ + if(!deterministic) + { + mean = arma::mean(input, 1); + variance = arma::var(input, 1, 1); + } + + output = beta + (gamma % (input - mean)) / arma::sqrt(variance + eps); + +} + +template +template +void BatchNorm::Backward( + const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) +{ + size_t n = input.n_cols; + + g = (1/n) * gamma % arma::pow(variance + eps, -0.5) % + (n * gy - arma::sum(gy, 1) - (input - mean) % arma::pow(variance + eps, -1.0) % arma::sum(gy % (input - mean), 1)); +} + +template +template +void BatchNorm::Gradient( + const arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& gradient) +{ + gradient.submat(0, 0, gamma.n_elem - 1, 0) = arma::sum((input - mean) % arma::pow(variance + eps, -0.5) % error, 1); + gradient.submat(gamma.n_elem, 0, gradient.n_elem - 1, 0) = arma::sum(error, 1); +} + +template +template +void BatchNorm::Serialize( + Archive& ar, const unsigned int /* version */) +{ + ar & data::CreateNVP(gamma, "gamma"); + ar & data::CreateNVP(beta, "beta"); + ar & data::CreateNVP(trainingMean, "trainingMean"); + ar & data::CreateNVP(trainingVariance, "trainingVariance"); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 867c665156..f00fa0a566 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -36,6 +36,7 @@ #include #include #include +#include // Convolution modules. #include @@ -45,6 +46,10 @@ namespace mlpack { namespace ann { + +template class AddMerge; +template class BatchNorm; +template class Concat; template class DropConnect; template class Glimpse; template class Linear; @@ -108,6 +113,7 @@ using LayerTypes = boost::variant< BaseLayer*, BaseLayer*, BaseLayer*, + BatchNorm*, BilinearInterpolation*, Concat*, ConcatPerformance, From a70abebb1a4f928707f39d4e1e4c7d7f421749e3 Mon Sep 17 00:00:00 2001 From: Praveen Ch Date: Thu, 9 Mar 2017 23:32:26 +1100 Subject: [PATCH 51/79] Correct forward pass --- src/mlpack/methods/ann/layer/batchnorm.hpp | 17 ++++++-- .../methods/ann/layer/batchnorm_impl.hpp | 39 +++++++++++++------ 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batchnorm.hpp b/src/mlpack/methods/ann/layer/batchnorm.hpp index be984c485a..452d26cde4 100644 --- a/src/mlpack/methods/ann/layer/batchnorm.hpp +++ b/src/mlpack/methods/ann/layer/batchnorm.hpp @@ -29,7 +29,7 @@ class BatchNorm BatchNorm(); - BatchNorm(const size_t size); + BatchNorm(const size_t size, const double eps); void Reset(); @@ -71,6 +71,15 @@ class BatchNorm //! Modify the gradient. OutputDataType& Gradient() { return gradient; } + OutputDataType& Mean() { return mean; } + + OutputDataType& Variance() { return variance; } + + OutputDataType& Gamma() { return gamma; } + + OutputDataType& Beta() { return beta; } + + template void Serialize(Archive& ar, const unsigned int /* version */); @@ -91,11 +100,13 @@ class BatchNorm OutputDataType mean; - OutputDataType trainingMean; + // OutputDataType trainingMean; OutputDataType variance; - OutputDataType trainingVariance; + // OutputDataType trainingVariance; + + arma::running_stat_vec stats; OutputDataType gradient; diff --git a/src/mlpack/methods/ann/layer/batchnorm_impl.hpp b/src/mlpack/methods/ann/layer/batchnorm_impl.hpp index 94c6d9f93e..8c33bc810e 100644 --- a/src/mlpack/methods/ann/layer/batchnorm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batchnorm_impl.hpp @@ -27,7 +27,7 @@ BatchNorm::BatchNorm() template BatchNorm::BatchNorm( - const size_t size) : size(size) + const size_t size, const double eps = 0.001) : size(size), eps(eps) { weights.set_size(size + size, 1); } @@ -36,10 +36,12 @@ BatchNorm::BatchNorm( template void BatchNorm::Reset() { - // variance = arma::mat(variance.memptr(), size, 1, false, false); - // mean = arma::mat(mean.memptr(), size, 1, false, false); gamma = arma::mat(weights.memptr(), size, 1, false, false); beta = arma::mat(weights.memptr() + gamma.n_elem, size, 1, false, false); + deterministic = false; + gamma.fill(1.0); + beta.fill(0.0); + stats.reset(); } template @@ -47,14 +49,29 @@ template void BatchNorm::Forward( const arma::Mat&& input, arma::Mat&& output) { - if(!deterministic) + // if(!deterministic) + // { + // mean = arma::mean(input, 1); + // variance = arma::var(input, 1, 1); + // } + + // output = beta + (gamma % (input - mean)) / arma::sqrt(variance + eps); + output.reshape(input.n_rows, input.n_cols); + + for (size_t i = 0; i < output.n_rows; i++) { - mean = arma::mean(input, 1); - variance = arma::var(input, 1, 1); + arma::mat inpRow = input.row(i); + + output.row(i) = (inpRow - arma::as_scalar(arma::mean(inpRow,1))); + + output.row(i) /= (arma::as_scalar(arma::sqrt(arma::var(inpRow,1, 1)) + + arma::as_scalar(eps))); + + output.row(i) *= arma::as_scalar(gamma.row(i)); + + output.row(i) += arma::as_scalar(beta.row(i)); + } - - output = beta + (gamma % (input - mean)) / arma::sqrt(variance + eps); - } template @@ -86,8 +103,8 @@ void BatchNorm::Serialize( { ar & data::CreateNVP(gamma, "gamma"); ar & data::CreateNVP(beta, "beta"); - ar & data::CreateNVP(trainingMean, "trainingMean"); - ar & data::CreateNVP(trainingVariance, "trainingVariance"); + ar & data::CreateNVP(stats.mean(), "trainingMean"); + ar & data::CreateNVP(stats.var(1), "trainingVariance"); } } // namespace ann From af7a45a244dcd1d578f7e91977de9f050f2f0d87 Mon Sep 17 00:00:00 2001 From: Praveen Ch Date: Tue, 21 Mar 2017 02:44:41 +0530 Subject: [PATCH 52/79] Completed the layer with documentation --- src/mlpack/methods/ann/layer/batchnorm.hpp | 129 ++++++++++++++---- .../methods/ann/layer/batchnorm_impl.hpp | 83 ++++++++--- 2 files changed, 161 insertions(+), 51 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batchnorm.hpp b/src/mlpack/methods/ann/layer/batchnorm.hpp index 452d26cde4..c9a6194a5e 100644 --- a/src/mlpack/methods/ann/layer/batchnorm.hpp +++ b/src/mlpack/methods/ann/layer/batchnorm.hpp @@ -1,6 +1,6 @@ /** * @file batchnorm.hpp - * @author Marcus Edel + * @author Praveen Ch * * Definition of the Batch Normalisation layer class as proposed by Ioffe et.al * @@ -9,6 +9,7 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ + #ifndef MLPACK_METHODS_ANN_LAYER_BATCHNORM_HPP #define MLPACK_METHODS_ANN_LAYER_BATCHNORM_HPP @@ -19,6 +20,36 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { +/** + * Implementation of the Batch Normalisation layer class. The layer tranforms + * the input data into zero mean and unit variance and then scales and shifts + * the data by parameters, gamma and beta respectively. These parameters are + * learnt by the network. + * + * If deterministic is false (training), the mean and variance over the batch is + * calculated and the data is normalized. If it is set to true (testing) then + * the mean and variance accrued over the training set is used. + * + * For more information, refer to the following paper, + * + * @code + * @article{DBLP:journals/corr/IoffeS15, + * author = {Sergey Ioffe and + * Christian Szegedy}, + * title = {Batch Normalization: Accelerating Deep Network Training by + * Reducing Internal Covariate Shift}, + * journal = {CoRR}, + * volume = {abs/1502.03167} + * } + * + * @endcode + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ + template < typename InputDataType = arma::mat, typename OutputDataType = arma::mat @@ -26,21 +57,52 @@ template < class BatchNorm { public: - + //! Creating BatchNorm object. BatchNorm(); + /** + * Create the BatchNorm layer object for a specified number of input units. + * + * @param size The number of input units. + * @param eps The epsilon added to variance to ensure numerical stability. + */ BatchNorm(const size_t size, const double eps); + /** + * Reset the layer parameters + */ void Reset(); + /** + * Forward pass of the Batch Normalization layer. Transforms the input data + * into zero mean and unit variance, scales the data by a factor gamma and + * shifts it by beta. + * + * @param input Input data for the layer + * @param output Resulting output activations. + */ template void Forward(const arma::Mat&& input, arma::Mat&& output); + /** + * Backward pass through the layer. + * + * @param input The input activations + * @param gy The backpropagated error. + * @param g The calculated gradient. + */ template void Backward(const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g); + /** + * Calculate the gradient using the output delta and the input activations. + * + * @param input The input activations + * @param error The calculated error + * @param gradient The calculated gradient. + */ template void Gradient(const arma::Mat&& input, arma::Mat&& error, @@ -71,57 +133,66 @@ class BatchNorm //! Modify the gradient. OutputDataType& Gradient() { return gradient; } - OutputDataType& Mean() { return mean; } - - OutputDataType& Variance() { return variance; } - - OutputDataType& Gamma() { return gamma; } - - OutputDataType& Beta() { return beta; } - + //! Get the value of deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of deterministic parameter. + bool& Deterministic() { return deterministic; } + /** + * Serialize the layer + */ template void Serialize(Archive& ar, const unsigned int /* version */); private: - - OutputDataType gamma; - - OutputDataType beta; - - OutputDataType weights; - - double eps; - - bool deterministic; - + //! Locally-stored number of input units. size_t size; + //! Locally-stored epsilon value. + double eps; + + //! Locally-stored scale parameter. + OutputDataType gamma; + + //! Locally-stored shift parameter. + OutputDataType beta; + + //! Locally-stored weight object. + OutputDataType weights; + + /** + * If true then mean and variance over the training set will be considered + * instead of being calculated over the batch. + */ + bool deterministic; + + //! Locally-stored mean object. OutputDataType mean; - // OutputDataType trainingMean; - + //! Locally-stored variance object. OutputDataType variance; - // OutputDataType trainingVariance; - + //! Locally-stored running statistics object. arma::running_stat_vec stats; + //! Locally-stored gradient object. OutputDataType gradient; + //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. + //! Locally-stored input parameter object. InputDataType inputParameter; //! Locally-stored output parameter object. OutputDataType outputParameter; -}; +}; // class BatchNorm -} -} +} // namespace ann +} // namespace mlpack +// Include the implementation. #include "batchnorm_impl.hpp" #endif diff --git a/src/mlpack/methods/ann/layer/batchnorm_impl.hpp b/src/mlpack/methods/ann/layer/batchnorm_impl.hpp index 8c33bc810e..7475b808f6 100644 --- a/src/mlpack/methods/ann/layer/batchnorm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batchnorm_impl.hpp @@ -13,11 +13,11 @@ #ifndef MLPACK_METHODS_ANN_LAYER_BATCHNORM_IMPL_HPP #define MLPACK_METHODS_ANN_LAYER_BATCHNORM_IMPL_HPP - +// In case it is not included. #include "batchnorm.hpp" namespace mlpack { -namespace ann { +namespace ann { /** Artificial Neural Network. */ template BatchNorm::BatchNorm() @@ -49,28 +49,43 @@ template void BatchNorm::Forward( const arma::Mat&& input, arma::Mat&& output) { - // if(!deterministic) - // { - // mean = arma::mean(input, 1); - // variance = arma::var(input, 1, 1); - // } - - // output = beta + (gamma % (input - mean)) / arma::sqrt(variance + eps); output.reshape(input.n_rows, input.n_cols); - for (size_t i = 0; i < output.n_rows; i++) + // Mean and variance over the entire training set will be used to compute + // the forward pass when deterministic is set to true. + if (deterministic) { - arma::mat inpRow = input.row(i); + for (size_t i = 0; i < output.n_cols; i++) + { + output.col(i) = input.col(i) - stats.mean().col(i); + } - output.row(i) = (inpRow - arma::as_scalar(arma::mean(inpRow,1))); - - output.row(i) /= (arma::as_scalar(arma::sqrt(arma::var(inpRow,1, 1)) + - arma::as_scalar(eps))); - - output.row(i) *= arma::as_scalar(gamma.row(i)); - - output.row(i) += arma::as_scalar(beta.row(i)); + for (size_t i = 0; i < output.n_rows; i++) + { + output.row(i) *= arma::as_scalar(gamma.row(i)); + output.row(i) /= arma::as_scalar(arma::sqrt(stats.var(1).row(i) + eps)); + output.row(i) += arma::as_scalar(beta.row(i)); + } + + } + else + { + mean = arma::mean(input, 1); + variance = arma::var(input, 1, 1); + + for (size_t i = 0; i < output.n_cols; i++) + { + output.col(i) = input.col(i) - mean; + stats(input.col(i)); + } + for (size_t i = 0; i < output.n_rows; i++) + { + output.row(i) = (input.row(i) - arma::as_scalar(arma::mean(input.row(i), 1))); + output.row(i) /= (arma::as_scalar(arma::sqrt(variance.row(i) + eps))); + output.row(i) *= arma::as_scalar(gamma.row(i)); + output.row(i) += arma::as_scalar(beta.row(i)); + } } } @@ -81,8 +96,23 @@ void BatchNorm::Backward( { size_t n = input.n_cols; - g = (1/n) * gamma % arma::pow(variance + eps, -0.5) % - (n * gy - arma::sum(gy, 1) - (input - mean) % arma::pow(variance + eps, -1.0) % arma::sum(gy % (input - mean), 1)); + g.reshape(input.n_rows, input.n_cols); + + for (size_t i = 0; i < input.n_rows; ++i) + { + mean = arma::mean(input.row(i), 1); + variance = arma::var(input.row(i), 1, 1); + + g.row(i) = -(input.row(i) - arma::as_scalar(mean)); + g.row(i) *= arma::as_scalar(arma::sum(gy.row(i) % + (input.row(i) - arma::as_scalar(mean)), 1)); + g.row(i) /= (arma::as_scalar(variance + eps)); + g.row(i) += (n * gy.row(i) - arma::as_scalar(arma::sum(gy.row(i),1))); + g.row(i) *= (1.0 / n) * arma::as_scalar(gamma.row(i)); + g.row(i) /= (arma::as_scalar(arma::sqrt(variance + eps))); + + } + } template @@ -92,7 +122,16 @@ void BatchNorm::Gradient( arma::Mat&& error, arma::Mat&& gradient) { - gradient.submat(0, 0, gamma.n_elem - 1, 0) = arma::sum((input - mean) % arma::pow(variance + eps, -0.5) % error, 1); + arma::mat normalized(input.n_rows, input.n_cols); + gradient.reshape(size + size, 1); + + for (size_t i = 0; i < normalized.n_rows; i++) + { + normalized.row(i) = (input.row(i) - arma::as_scalar(arma::mean(input.row(i), 1))); + normalized.row(i) /= (arma::as_scalar(arma::sqrt(variance.row(i) + eps))); + } + + gradient.submat(0, 0, gamma.n_elem - 1, 0) = arma::sum(normalized % error, 1); gradient.submat(gamma.n_elem, 0, gradient.n_elem - 1, 0) = arma::sum(error, 1); } From 021c1a7263b327045625f354cd1df8ff14a7ae3d Mon Sep 17 00:00:00 2001 From: Praveen Ch Date: Tue, 21 Mar 2017 16:01:51 +0530 Subject: [PATCH 53/79] Fixed minor bug --- src/mlpack/methods/ann/layer/batchnorm_impl.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batchnorm_impl.hpp b/src/mlpack/methods/ann/layer/batchnorm_impl.hpp index 7475b808f6..32d89a934f 100644 --- a/src/mlpack/methods/ann/layer/batchnorm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batchnorm_impl.hpp @@ -127,8 +127,9 @@ void BatchNorm::Gradient( for (size_t i = 0; i < normalized.n_rows; i++) { - normalized.row(i) = (input.row(i) - arma::as_scalar(arma::mean(input.row(i), 1))); - normalized.row(i) /= (arma::as_scalar(arma::sqrt(variance.row(i) + eps))); + normalized.row(i) = input.row(i) - arma::as_scalar(arma::mean(input.row(i), 1)); + normalized.row(i) /= + arma::as_scalar(arma::sqrt(arma::var(input.row(i), 1, 1) + eps)); } gradient.submat(0, 0, gamma.n_elem - 1, 0) = arma::sum(normalized % error, 1); From 654151858f8038534cb1d3c022b4977a9956b2e5 Mon Sep 17 00:00:00 2001 From: Praveen Ch Date: Tue, 21 Mar 2017 17:26:36 +0530 Subject: [PATCH 54/79] Minor Forward pass modification --- src/mlpack/methods/ann/layer/batchnorm.hpp | 6 ++++++ .../methods/ann/layer/batchnorm_impl.hpp | 19 ++++--------------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batchnorm.hpp b/src/mlpack/methods/ann/layer/batchnorm.hpp index c9a6194a5e..678eb6ad90 100644 --- a/src/mlpack/methods/ann/layer/batchnorm.hpp +++ b/src/mlpack/methods/ann/layer/batchnorm.hpp @@ -138,6 +138,12 @@ class BatchNorm //! Modify the value of deterministic parameter. bool& Deterministic() { return deterministic; } + //! Get the mean over the training data. + OutputDataType TrainingMean() { return stats.mean(); } + + //! Get the variance over the training data. + OutputDataType TrainingVariance() { return stats.var(1); } + /** * Serialize the layer diff --git a/src/mlpack/methods/ann/layer/batchnorm_impl.hpp b/src/mlpack/methods/ann/layer/batchnorm_impl.hpp index 32d89a934f..fed4c056fc 100644 --- a/src/mlpack/methods/ann/layer/batchnorm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batchnorm_impl.hpp @@ -55,18 +55,8 @@ void BatchNorm::Forward( // the forward pass when deterministic is set to true. if (deterministic) { - for (size_t i = 0; i < output.n_cols; i++) - { - output.col(i) = input.col(i) - stats.mean().col(i); - } - - for (size_t i = 0; i < output.n_rows; i++) - { - output.row(i) *= arma::as_scalar(gamma.row(i)); - output.row(i) /= arma::as_scalar(arma::sqrt(stats.var(1).row(i) + eps)); - output.row(i) += arma::as_scalar(beta.row(i)); - } - + mean = stats.mean(); + variance = stats.var(1); } else { @@ -75,19 +65,18 @@ void BatchNorm::Forward( for (size_t i = 0; i < output.n_cols; i++) { - output.col(i) = input.col(i) - mean; stats(input.col(i)); } + } for (size_t i = 0; i < output.n_rows; i++) { - output.row(i) = (input.row(i) - arma::as_scalar(arma::mean(input.row(i), 1))); + output.row(i) = input.row(i) - arma::as_scalar(mean.row(i)); output.row(i) /= (arma::as_scalar(arma::sqrt(variance.row(i) + eps))); output.row(i) *= arma::as_scalar(gamma.row(i)); output.row(i) += arma::as_scalar(beta.row(i)); } } -} template template From 321c6d8b83316ba9278c3947f51c852f0b4b0fa2 Mon Sep 17 00:00:00 2001 From: Praveen Ch Date: Tue, 21 Mar 2017 17:27:03 +0530 Subject: [PATCH 55/79] Added tests for BatchNorm Layer --- src/mlpack/tests/ann_layer_test.cpp | 99 +++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index caaa88fe64..942c3f435b 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -1,6 +1,7 @@ /** * @file ann_layer_test.cpp * @author Marcus Edel + * @author Praveen Ch * * Tests the ann layer modules. * @@ -312,6 +313,104 @@ BOOST_AUTO_TEST_CASE(JacobianConstantLayerTest) } } +/** + * Tests the BatchNorm Layer, compares the layers parameters with + * the values from another implementation. + */ +BOOST_AUTO_TEST_CASE(BatchNormTest) +{ + arma::mat dataset, output; + data::Load("iris.csv", dataset, true); + + arma::mat input = dataset.submat(0, 0, dataset.n_rows-1, 2); + + BatchNorm<> model(input.n_rows); + model.Reset(); + + // Non-Deteministic Forward Pass Test. + model.Deterministic() = false; + model.Forward(std::move(input), std::move(output)); + + arma::mat result; + result << 1.20240722e+00 << 5.33976074e-15 << -1.20240722e+00 << arma::endr + << 1.28267074e+00 << -1.12233689e+00 << -1.60333842e-01 << arma::endr + << 5.87220220e-01 << 5.87220220e-01 << -1.17444044e+00 << arma::endr + << 0.00000000e+00 << 0.00000000e+00 << 0.00000000e+00 << arma::endr; + + CheckMatrices(output, result); + result.clear(); + + // Backward Pass Test. + arma::mat gy; + gy << 0.8402 << 0.9116 << 0.2778 << arma::endr + << 0.3944 << 0.1976 << 0.5540 << arma::endr + << 0.7831 << 0.3352 << 0.4774 << arma::endr + << 0.7984 << 0.7682 << 0.6289 << arma::endr; + + model.Backward(std::move(input), std::move(gy), std::move(output)); + + result << -0.64550918 << 1.41322929 << -0.76772011 << arma::endr + << -0.34197354 << -0.5355513 << 0.87752484 << arma::endr + << 4.09422086 << -3.79625723 << -0.29796364 << arma::endr + << 2.10502283 << 1.15001498 << -3.2550378 << arma::endr; + + CheckMatrices(output, result); + result.clear(); + + // Gradient Test. + model.Gradient(std::move(input), std::move(gy), std::move(output)); + + result << 0.67623382 << arma::endr + << 0.19528662 << arma::endr + << 0.09601051 << arma::endr + << 0.00000000 << arma::endr + << 2.0296 << arma::endr + << 1.146 << arma::endr + << 1.5957 << arma::endr + << 2.1955 << arma::endr; + + CheckMatrices(output, result); + result.clear(); + + // Deterministic Forward Pass test. + input = dataset.submat(0, 3, dataset.n_rows-1, 5); + model.Forward(std::move(input), std::move(output)); + + input = dataset.submat(0, 6, dataset.n_rows-1, 8); + model.Forward(std::move(input), std::move(output)); + + output = model.TrainingMean(); + result << 4.85555556 << arma::endr + << 3.33333333 << arma::endr + << 1.44444444 << arma::endr + << 0.23333333 << arma::endr; + + CheckMatrices(output, result); + result.clear(); + + output = model.TrainingVariance(); + result << 0.08469136 << arma::endr + << 0.08888889 << arma::endr + << 0.01135802 << arma::endr + << 0.00444444 << arma::endr; + + CheckMatrices(output, result, 2e-4); + result.clear(); + + input = dataset.submat(0, 0, dataset.n_rows-1, 2); + + model.Deterministic() = true; + model.Forward(std::move(input), std::move(output)); + + result << 0.83504842 << 0.15182699 << -0.53139445 << arma::endr + << 0.55589881 << -1.11179762 << -0.44471905 << arma::endr + << -0.39980015 << -0.39980015 << -1.29935049 << arma::endr + << -0.45175395 << -0.45175395 << -0.45175395 << arma::endr; + + CheckMatrices(output, result); + +} + /** * Simple dropout module test. */ From 8bfe4c8e7e3c3f09cfec9174b471ec28d42612ee Mon Sep 17 00:00:00 2001 From: Praveen Ch Date: Thu, 23 Mar 2017 18:49:30 +0530 Subject: [PATCH 56/79] Added typecast to rectify Windows build error --- src/mlpack/tests/ann_layer_test.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 942c3f435b..9c46fba8db 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -324,7 +324,9 @@ BOOST_AUTO_TEST_CASE(BatchNormTest) arma::mat input = dataset.submat(0, 0, dataset.n_rows-1, 2); - BatchNorm<> model(input.n_rows); + size_t numUnits = input.n_rows; + + BatchNorm<> model(numUnits); model.Reset(); // Non-Deteministic Forward Pass Test. From 15f9961c46e730ccf6c7cb0f23e5854eb8fcef56 Mon Sep 17 00:00:00 2001 From: Praveen Ch Date: Fri, 24 Mar 2017 00:13:32 +0530 Subject: [PATCH 57/79] Removed default constructor --- src/mlpack/methods/ann/layer/batchnorm.hpp | 5 +---- src/mlpack/methods/ann/layer/batchnorm_impl.hpp | 6 ------ 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batchnorm.hpp b/src/mlpack/methods/ann/layer/batchnorm.hpp index 678eb6ad90..ee3cd9b421 100644 --- a/src/mlpack/methods/ann/layer/batchnorm.hpp +++ b/src/mlpack/methods/ann/layer/batchnorm.hpp @@ -57,10 +57,7 @@ template < class BatchNorm { public: - //! Creating BatchNorm object. - BatchNorm(); - - /** + /** * Create the BatchNorm layer object for a specified number of input units. * * @param size The number of input units. diff --git a/src/mlpack/methods/ann/layer/batchnorm_impl.hpp b/src/mlpack/methods/ann/layer/batchnorm_impl.hpp index fed4c056fc..bc37e1653d 100644 --- a/src/mlpack/methods/ann/layer/batchnorm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batchnorm_impl.hpp @@ -19,12 +19,6 @@ namespace mlpack { namespace ann { /** Artificial Neural Network. */ -template -BatchNorm::BatchNorm() -{ - // Nothing to do here. -} - template BatchNorm::BatchNorm( const size_t size, const double eps = 0.001) : size(size), eps(eps) From 38a4ab2af326116a63bd5423f0306a4ae0ab2828 Mon Sep 17 00:00:00 2001 From: Praveen Ch Date: Fri, 24 Mar 2017 08:41:47 +0530 Subject: [PATCH 58/79] Moved the headers --- src/mlpack/methods/ann/layer/layer.hpp | 1 + src/mlpack/methods/ann/layer/layer_types.hpp | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/layer/layer.hpp b/src/mlpack/methods/ann/layer/layer.hpp index f47eadfd2f..d72b473668 100644 --- a/src/mlpack/methods/ann/layer/layer.hpp +++ b/src/mlpack/methods/ann/layer/layer.hpp @@ -13,6 +13,7 @@ #define MLPACK_METHODS_ANN_LAYER_LAYER_HPP #include "add_merge.hpp" +#include "batchnorm.hpp" #include "concat_performance.hpp" #include "convolution.hpp" #include "dropconnect.hpp" diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index f00fa0a566..c27b1449c9 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -36,7 +36,6 @@ #include #include #include -#include // Convolution modules. #include From 24579155490828261be938cc17aa508583dbeb33 Mon Sep 17 00:00:00 2001 From: Praveen Ch Date: Sat, 25 Mar 2017 00:40:17 +0530 Subject: [PATCH 59/79] Made the constructor explicit --- src/mlpack/methods/ann/layer/batchnorm.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/layer/batchnorm.hpp b/src/mlpack/methods/ann/layer/batchnorm.hpp index ee3cd9b421..7200804d2a 100644 --- a/src/mlpack/methods/ann/layer/batchnorm.hpp +++ b/src/mlpack/methods/ann/layer/batchnorm.hpp @@ -63,7 +63,7 @@ class BatchNorm * @param size The number of input units. * @param eps The epsilon added to variance to ensure numerical stability. */ - BatchNorm(const size_t size, const double eps); + explicit BatchNorm(const size_t size, const double eps); /** * Reset the layer parameters From 3c2081af0f86f88d1884ec83906306a6c3564e84 Mon Sep 17 00:00:00 2001 From: Praveen Ch Date: Sun, 26 Mar 2017 11:36:42 +0530 Subject: [PATCH 60/79] Moved the default declaration to the header file --- src/mlpack/methods/ann/layer/batchnorm.hpp | 2 +- src/mlpack/methods/ann/layer/batchnorm_impl.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batchnorm.hpp b/src/mlpack/methods/ann/layer/batchnorm.hpp index 7200804d2a..6520cc6b5f 100644 --- a/src/mlpack/methods/ann/layer/batchnorm.hpp +++ b/src/mlpack/methods/ann/layer/batchnorm.hpp @@ -63,7 +63,7 @@ class BatchNorm * @param size The number of input units. * @param eps The epsilon added to variance to ensure numerical stability. */ - explicit BatchNorm(const size_t size, const double eps); + BatchNorm(const size_t size, const double eps = 0.001); /** * Reset the layer parameters diff --git a/src/mlpack/methods/ann/layer/batchnorm_impl.hpp b/src/mlpack/methods/ann/layer/batchnorm_impl.hpp index bc37e1653d..c50249862d 100644 --- a/src/mlpack/methods/ann/layer/batchnorm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batchnorm_impl.hpp @@ -21,7 +21,7 @@ namespace ann { /** Artificial Neural Network. */ template BatchNorm::BatchNorm( - const size_t size, const double eps = 0.001) : size(size), eps(eps) + const size_t size, const double eps) : size(size), eps(eps) { weights.set_size(size + size, 1); } From 6e13ab585fce9fb75f193ba3529344d3bb414bd0 Mon Sep 17 00:00:00 2001 From: Praveen Ch Date: Fri, 14 Apr 2017 12:02:48 +0530 Subject: [PATCH 61/79] Added link to Python implementation --- src/mlpack/tests/ann_layer_test.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 9c46fba8db..7ddb93dd61 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -315,7 +315,8 @@ BOOST_AUTO_TEST_CASE(JacobianConstantLayerTest) /** * Tests the BatchNorm Layer, compares the layers parameters with - * the values from another implementation. + * the values from another implementation. + * Link to the implementation - http://cthorey.github.io./backpropagation/ */ BOOST_AUTO_TEST_CASE(BatchNormTest) { From fc8b896bd1517ea8c4b735b2762011766b706c83 Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Fri, 9 Mar 2018 03:17:52 +0530 Subject: [PATCH 62/79] Implement Batchnorm layer and test --- src/mlpack/methods/ann/layer/CMakeLists.txt | 4 +- .../layer/{batchnorm.hpp => batch_norm.hpp} | 41 +++--- ...batchnorm_impl.hpp => batch_norm_impl.hpp} | 84 ++++++------ src/mlpack/methods/ann/layer/layer.hpp | 2 +- src/mlpack/methods/ann/layer/layer_types.hpp | 3 +- src/mlpack/tests/ann_layer_test.cpp | 125 ++++++++++++++++++ 6 files changed, 189 insertions(+), 70 deletions(-) rename src/mlpack/methods/ann/layer/{batchnorm.hpp => batch_norm.hpp} (88%) rename src/mlpack/methods/ann/layer/{batchnorm_impl.hpp => batch_norm_impl.hpp} (55%) diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt index c749376196..38dfa4377b 100644 --- a/src/mlpack/methods/ann/layer/CMakeLists.txt +++ b/src/mlpack/methods/ann/layer/CMakeLists.txt @@ -8,8 +8,8 @@ set(SOURCES base_layer.hpp bilinear_interpolation.hpp bilinear_interpolation_impl.hpp - batchnorm.hpp - batchnorm_impl.hpp + batch_norm.hpp + batch_norm_impl.hpp concat.hpp concat_impl.hpp concat_performance.hpp diff --git a/src/mlpack/methods/ann/layer/batchnorm.hpp b/src/mlpack/methods/ann/layer/batch_norm.hpp similarity index 88% rename from src/mlpack/methods/ann/layer/batchnorm.hpp rename to src/mlpack/methods/ann/layer/batch_norm.hpp index 6520cc6b5f..31bb00f4ff 100644 --- a/src/mlpack/methods/ann/layer/batchnorm.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm.hpp @@ -1,8 +1,9 @@ /** * @file batchnorm.hpp * @author Praveen Ch + * @author Manthan-R-Sheth * - * Definition of the Batch Normalisation layer class as proposed by Ioffe et.al + * Definition of the Batch Normalisation layer class * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -15,19 +16,17 @@ #include -#include "layer_types.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * Implementation of the Batch Normalisation layer class. The layer tranforms + * Declaration of the Batch Normalisation layer class. The layer tranforms * the input data into zero mean and unit variance and then scales and shifts - * the data by parameters, gamma and beta respectively. These parameters are + * the data by parameters, gamma and beta respectively. These parameters are * learnt by the network. * * If deterministic is false (training), the mean and variance over the batch is - * calculated and the data is normalized. If it is set to true (testing) then + * calculated and the data is normalized. If it is set to true (testing) then * the mean and variance accrued over the training set is used. * * For more information, refer to the following paper, @@ -36,14 +35,14 @@ namespace ann /** Artificial Neural Network. */ { * @article{DBLP:journals/corr/IoffeS15, * author = {Sergey Ioffe and * Christian Szegedy}, - * title = {Batch Normalization: Accelerating Deep Network Training by + * title = {Batch Normalization: Accelerating Deep Network Training by * Reducing Internal Covariate Shift}, * journal = {CoRR}, * volume = {abs/1502.03167} * } * * @endcode - * + * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, @@ -51,18 +50,21 @@ namespace ann /** Artificial Neural Network. */ { */ template < - typename InputDataType = arma::mat, - typename OutputDataType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > class BatchNorm { public: - /** - * Create the BatchNorm layer object for a specified number of input units. - * - * @param size The number of input units. - * @param eps The epsilon added to variance to ensure numerical stability. - */ + //! Create the BatchNorm object. + BatchNorm(); + + /** + * Create the BatchNorm layer object for a specified number of input units. + * + * @param size The number of input units. + * @param eps The epsilon added to variance to ensure numerical stability. + */ BatchNorm(const size_t size, const double eps = 0.001); /** @@ -141,12 +143,11 @@ class BatchNorm //! Get the variance over the training data. OutputDataType TrainingVariance() { return stats.var(1); } - /** * Serialize the layer */ template - void Serialize(Archive& ar, const unsigned int /* version */); + void serialize(Archive& ar, const unsigned int /* version */); private: //! Locally-stored number of input units. @@ -161,7 +162,7 @@ class BatchNorm //! Locally-stored shift parameter. OutputDataType beta; - //! Locally-stored weight object. + //! Locally-stored parameters. OutputDataType weights; /** @@ -196,6 +197,6 @@ class BatchNorm } // namespace mlpack // Include the implementation. -#include "batchnorm_impl.hpp" +#include "batch_norm_impl.hpp" #endif diff --git a/src/mlpack/methods/ann/layer/batchnorm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp similarity index 55% rename from src/mlpack/methods/ann/layer/batchnorm_impl.hpp rename to src/mlpack/methods/ann/layer/batch_norm_impl.hpp index c50249862d..58cd633d4c 100644 --- a/src/mlpack/methods/ann/layer/batchnorm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -1,9 +1,10 @@ /** * @file batchnorm_impl.hpp * @author Praveen Ch + * @author Manthan-R-Sheth * * Implementation of the Batch Normalization Layer. - * + * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the * 3-clause BSD license along with mlpack. If not, see @@ -14,14 +15,20 @@ #define MLPACK_METHODS_ANN_LAYER_BATCHNORM_IMPL_HPP // In case it is not included. -#include "batchnorm.hpp" +#include "batch_norm.hpp" namespace mlpack { namespace ann { /** Artificial Neural Network. */ +template +BatchNorm::BatchNorm() +{ + // Nothing to do here. +} + template BatchNorm::BatchNorm( - const size_t size, const double eps) : size(size), eps(eps) + const size_t size, const double eps) : size(size), eps(eps) { weights.set_size(size + size, 1); } @@ -41,7 +48,7 @@ void BatchNorm::Reset() template template void BatchNorm::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat&& input, arma::Mat&& output) { output.reshape(input.n_rows, input.n_cols); @@ -50,84 +57,69 @@ void BatchNorm::Forward( if (deterministic) { mean = stats.mean(); - variance = stats.var(1); + variance = stats.var(1); } else { mean = arma::mean(input, 1); variance = arma::var(input, 1, 1); - + for (size_t i = 0; i < output.n_cols; i++) { stats(input.col(i)); } } - for (size_t i = 0; i < output.n_rows; i++) - { - output.row(i) = input.row(i) - arma::as_scalar(mean.row(i)); - output.row(i) /= (arma::as_scalar(arma::sqrt(variance.row(i) + eps))); - output.row(i) *= arma::as_scalar(gamma.row(i)); - output.row(i) += arma::as_scalar(beta.row(i)); - } - } + output = input.each_col() - mean; + output.each_col() %= gamma/arma::sqrt(variance+eps); + output.each_col() += beta; +} template template void BatchNorm::Backward( - const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) { size_t n = input.n_cols; g.reshape(input.n_rows, input.n_cols); - for (size_t i = 0; i < input.n_rows; ++i) - { - mean = arma::mean(input.row(i), 1); - variance = arma::var(input.row(i), 1, 1); - - g.row(i) = -(input.row(i) - arma::as_scalar(mean)); - g.row(i) *= arma::as_scalar(arma::sum(gy.row(i) % - (input.row(i) - arma::as_scalar(mean)), 1)); - g.row(i) /= (arma::as_scalar(variance + eps)); - g.row(i) += (n * gy.row(i) - arma::as_scalar(arma::sum(gy.row(i),1))); - g.row(i) *= (1.0 / n) * arma::as_scalar(gamma.row(i)); - g.row(i) /= (arma::as_scalar(arma::sqrt(variance + eps))); - - } + mean = arma::mean(input, 1); + variance = arma::var(input, 1, 1); + arma::mat m = arma::sum(gy % (input.each_col() - mean), 1); + g = arma::repmat(m, 1, input.n_cols) % -(input.each_col() - mean); + g.each_col() %= 1.0/(variance + eps); + g += (n * gy - arma::repmat((arma::sum(gy, 1)), 1, input.n_cols)); + g.each_col() %= ((1.0 / n) * gamma); + g.each_col() %= (1.0/arma::sqrt(variance + eps)); } template template void BatchNorm::Gradient( - const arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient) + const arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& gradient) { arma::mat normalized(input.n_rows, input.n_cols); gradient.reshape(size + size, 1); - - for (size_t i = 0; i < normalized.n_rows; i++) - { - normalized.row(i) = input.row(i) - arma::as_scalar(arma::mean(input.row(i), 1)); - normalized.row(i) /= - arma::as_scalar(arma::sqrt(arma::var(input.row(i), 1, 1) + eps)); - } + + normalized = input.each_col() - arma::mean(input, 1) + / arma::sqrt(arma::var(input, 1, 1) + eps); gradient.submat(0, 0, gamma.n_elem - 1, 0) = arma::sum(normalized % error, 1); - gradient.submat(gamma.n_elem, 0, gradient.n_elem - 1, 0) = arma::sum(error, 1); + gradient.submat(gamma.n_elem, 0, gradient.n_elem - 1, 0) = + arma::sum(error, 1); } template template -void BatchNorm::Serialize( - Archive& ar, const unsigned int /* version */) +void BatchNorm::serialize( + Archive& ar, const unsigned int /* version */) { - ar & data::CreateNVP(gamma, "gamma"); - ar & data::CreateNVP(beta, "beta"); - ar & data::CreateNVP(stats.mean(), "trainingMean"); - ar & data::CreateNVP(stats.var(1), "trainingVariance"); + ar & BOOST_SERIALIZATION_NVP(gamma); + ar & BOOST_SERIALIZATION_NVP(beta); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/layer.hpp b/src/mlpack/methods/ann/layer/layer.hpp index d72b473668..f50b988be6 100644 --- a/src/mlpack/methods/ann/layer/layer.hpp +++ b/src/mlpack/methods/ann/layer/layer.hpp @@ -13,7 +13,7 @@ #define MLPACK_METHODS_ANN_LAYER_LAYER_HPP #include "add_merge.hpp" -#include "batchnorm.hpp" +#include "batch_norm.hpp" #include "concat_performance.hpp" #include "convolution.hpp" #include "dropconnect.hpp" diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index c27b1449c9..ab2b213380 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -17,6 +17,7 @@ // Layer modules. #include #include +#include #include #include #include @@ -113,7 +114,7 @@ using LayerTypes = boost::variant< BaseLayer*, BaseLayer*, BatchNorm*, - BilinearInterpolation*, +// BilinearInterpolation*, Concat*, ConcatPerformance, arma::mat, arma::mat>*, diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 7ddb93dd61..bc3af3fcf0 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -1419,4 +1419,129 @@ BOOST_AUTO_TEST_CASE(SimpleBilinearInterpolationLayerTest) arma::zeros(input.n_rows), 1e-12); } +/** + * Tests the BatchNorm Layer, compares the layers parameters with + * the values from another implementation. + * Link to the implementation - http://cthorey.github.io./backpropagation/ + */ +BOOST_AUTO_TEST_CASE(BatchNormTest) +{ + arma::mat input, output; + input << 5.1 << 3.5 << 1.4 << arma::endr + << 4.9 << 3.0 << 1.4 << arma::endr + << 4.7 << 3.2 << 1.3 << arma::endr; + + size_t numUnits = input.n_rows; + + BatchNorm<> model(numUnits); + model.Reset(); + + // Non-Deteministic Forward Pass Test. + model.Deterministic() = false; + model.Forward(std::move(input), std::move(output)); + arma::mat result; + result << 1.1658 << 0.1100 << -1.2758 << arma::endr + << 1.2579 << -0.0699 << -1.1880 << arma::endr + << 1.1737 << 0.0958 << -1.2695 << arma::endr; + + CheckMatrices(output, result, 1e-1); + result.clear(); + + // Backward Pass Test. + arma::mat gy; + gy << 0.8402 << 0.9116 << 0.2778 << arma::endr + << 0.3944 << 0.1976 << 0.5540 << arma::endr + << 0.7831 << 0.3352 << 0.4774 << arma::endr; + + model.Backward(std::move(input), std::move(gy), std::move(output)); + result << -0.0780 << 0.1376 << -0.0596 << arma::endr + << 0.0602 << -0.1317 << 0.0715 << arma::endr + << 0.0835 << -0.1493 << 0.0658 << arma::endr; + + CheckMatrices(output, result, 1e-1); + result.clear(); + + // Gradient Test. + model.Gradient(std::move(input), std::move(gy), std::move(output)); + result << 3.4003 << arma::endr + << 0.8183 << arma::endr + << 1.8574 << arma::endr + << 2.0296 << arma::endr + << 1.1460 << arma::endr + << 1.5957 << arma::endr; + + CheckMatrices(output, result, 1e-1); + result.clear(); + + // Deterministic Forward Pass test. + output = model.TrainingMean(); + result << 3.33333333 << arma::endr + << 3.1 << arma::endr + << 3.06666666 << arma::endr; + + CheckMatrices(output, result, 1e-1); + result.clear(); + + output = model.TrainingVariance(); + result << 2.2956 << arma::endr + << 2.0467 << arma::endr + << 1.9356 << arma::endr; + + CheckMatrices(output, result, 1e-1); + result.clear(); + + model.Deterministic() = true; + model.Forward(std::move(input), std::move(output)); + + result << 1.1658 << 0.1100 << -1.2757 << arma::endr + << 1.2579 << -0.0699 << -1.1880 << arma::endr + << 1.1737 << 0.0958 << -1.2695 << arma::endr; + + CheckMatrices(output, result, 1e-1); +} + +/** + * BatchNorm layer numerically gradient test. + */ +BOOST_AUTO_TEST_CASE(GradientBatchNormLayerTest) +{ + // Add function gradient instantiation. + struct GradientFunction + { + GradientFunction() + { + input = arma::randn(10, 256); + arma::mat target; + target.ones(1, 256); + + model = new FFN, NguyenWidrowInitialization>( + input, target); + model->Add >(); + model->Add >(10); + model->Add >(10, 2); + model->Add >(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + arma::mat output; + double error = model->Evaluate(model->Parameters(), 0, 256, false); + model->Gradient(model->Parameters(), 0, gradient, 256); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN, NguyenWidrowInitialization>* model; + arma::mat input, target; + } function; + + BOOST_REQUIRE_LE(CheckGradient(function), 1e-3); +} + BOOST_AUTO_TEST_SUITE_END(); From 0f7eb1c839d515d58f73258a9a0fdfe36544c571 Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Fri, 9 Mar 2018 03:24:02 +0530 Subject: [PATCH 63/79] Clean rebase issues --- src/mlpack/methods/ann/layer/batch_norm.hpp | 2 +- .../methods/ann/layer/batch_norm_impl.hpp | 2 +- src/mlpack/tests/ann_layer_test.cpp | 101 ------------------ 3 files changed, 2 insertions(+), 103 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batch_norm.hpp b/src/mlpack/methods/ann/layer/batch_norm.hpp index 31bb00f4ff..1bd4e1afb0 100644 --- a/src/mlpack/methods/ann/layer/batch_norm.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm.hpp @@ -1,5 +1,5 @@ /** - * @file batchnorm.hpp + * @file batch_norm.hpp * @author Praveen Ch * @author Manthan-R-Sheth * diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index 58cd633d4c..ea67409bd5 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -1,5 +1,5 @@ /** - * @file batchnorm_impl.hpp + * @file batch_norm_impl.hpp * @author Praveen Ch * @author Manthan-R-Sheth * diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index bc3af3fcf0..1dfd1cf680 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -313,107 +313,6 @@ BOOST_AUTO_TEST_CASE(JacobianConstantLayerTest) } } -/** - * Tests the BatchNorm Layer, compares the layers parameters with - * the values from another implementation. - * Link to the implementation - http://cthorey.github.io./backpropagation/ - */ -BOOST_AUTO_TEST_CASE(BatchNormTest) -{ - arma::mat dataset, output; - data::Load("iris.csv", dataset, true); - - arma::mat input = dataset.submat(0, 0, dataset.n_rows-1, 2); - - size_t numUnits = input.n_rows; - - BatchNorm<> model(numUnits); - model.Reset(); - - // Non-Deteministic Forward Pass Test. - model.Deterministic() = false; - model.Forward(std::move(input), std::move(output)); - - arma::mat result; - result << 1.20240722e+00 << 5.33976074e-15 << -1.20240722e+00 << arma::endr - << 1.28267074e+00 << -1.12233689e+00 << -1.60333842e-01 << arma::endr - << 5.87220220e-01 << 5.87220220e-01 << -1.17444044e+00 << arma::endr - << 0.00000000e+00 << 0.00000000e+00 << 0.00000000e+00 << arma::endr; - - CheckMatrices(output, result); - result.clear(); - - // Backward Pass Test. - arma::mat gy; - gy << 0.8402 << 0.9116 << 0.2778 << arma::endr - << 0.3944 << 0.1976 << 0.5540 << arma::endr - << 0.7831 << 0.3352 << 0.4774 << arma::endr - << 0.7984 << 0.7682 << 0.6289 << arma::endr; - - model.Backward(std::move(input), std::move(gy), std::move(output)); - - result << -0.64550918 << 1.41322929 << -0.76772011 << arma::endr - << -0.34197354 << -0.5355513 << 0.87752484 << arma::endr - << 4.09422086 << -3.79625723 << -0.29796364 << arma::endr - << 2.10502283 << 1.15001498 << -3.2550378 << arma::endr; - - CheckMatrices(output, result); - result.clear(); - - // Gradient Test. - model.Gradient(std::move(input), std::move(gy), std::move(output)); - - result << 0.67623382 << arma::endr - << 0.19528662 << arma::endr - << 0.09601051 << arma::endr - << 0.00000000 << arma::endr - << 2.0296 << arma::endr - << 1.146 << arma::endr - << 1.5957 << arma::endr - << 2.1955 << arma::endr; - - CheckMatrices(output, result); - result.clear(); - - // Deterministic Forward Pass test. - input = dataset.submat(0, 3, dataset.n_rows-1, 5); - model.Forward(std::move(input), std::move(output)); - - input = dataset.submat(0, 6, dataset.n_rows-1, 8); - model.Forward(std::move(input), std::move(output)); - - output = model.TrainingMean(); - result << 4.85555556 << arma::endr - << 3.33333333 << arma::endr - << 1.44444444 << arma::endr - << 0.23333333 << arma::endr; - - CheckMatrices(output, result); - result.clear(); - - output = model.TrainingVariance(); - result << 0.08469136 << arma::endr - << 0.08888889 << arma::endr - << 0.01135802 << arma::endr - << 0.00444444 << arma::endr; - - CheckMatrices(output, result, 2e-4); - result.clear(); - - input = dataset.submat(0, 0, dataset.n_rows-1, 2); - - model.Deterministic() = true; - model.Forward(std::move(input), std::move(output)); - - result << 0.83504842 << 0.15182699 << -0.53139445 << arma::endr - << 0.55589881 << -1.11179762 << -0.44471905 << arma::endr - << -0.39980015 << -0.39980015 << -1.29935049 << arma::endr - << -0.45175395 << -0.45175395 << -0.45175395 << arma::endr; - - CheckMatrices(output, result); - -} - /** * Simple dropout module test. */ From df990d55c0fcd9d0acd0c7b10eab5b495d348414 Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Fri, 9 Mar 2018 10:26:39 +0530 Subject: [PATCH 64/79] Remove redeclarations --- src/mlpack/methods/ann/layer/layer_types.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index ab2b213380..c1009ccfdb 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -47,9 +47,7 @@ namespace mlpack { namespace ann { -template class AddMerge; template class BatchNorm; -template class Concat; template class DropConnect; template class Glimpse; template class Linear; From b9b66bc27031c8e4ed409ec285b741d94fada479 Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Fri, 9 Mar 2018 19:43:56 +0530 Subject: [PATCH 65/79] Static analysis and style fix --- .../methods/ann/layer/batch_norm_impl.hpp | 28 +++++++++++-------- src/mlpack/methods/ann/layer/layer_types.hpp | 6 +++- src/mlpack/tests/ann_layer_test.cpp | 6 ++-- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index ea67409bd5..e96c34095b 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -21,19 +21,24 @@ namespace mlpack { namespace ann { /** Artificial Neural Network. */ template -BatchNorm::BatchNorm() +BatchNorm::BatchNorm(): + size(10), + eps(1e-7), + deterministic(false) { // Nothing to do here. } template BatchNorm::BatchNorm( - const size_t size, const double eps) : size(size), eps(eps) + const size_t size, const double eps): + size(size), + eps(eps), + deterministic(false) { weights.set_size(size + size, 1); } - template void BatchNorm::Reset() { @@ -48,7 +53,7 @@ void BatchNorm::Reset() template template void BatchNorm::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat&& input, arma::Mat&& output) { output.reshape(input.n_rows, input.n_cols); @@ -71,7 +76,7 @@ void BatchNorm::Forward( } output = input.each_col() - mean; - output.each_col() %= gamma/arma::sqrt(variance+eps); + output.each_col() %= gamma / arma::sqrt(variance+eps); output.each_col() += beta; } @@ -80,7 +85,6 @@ template void BatchNorm::Backward( const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) { - size_t n = input.n_cols; g.reshape(input.n_rows, input.n_cols); @@ -89,10 +93,10 @@ void BatchNorm::Backward( arma::mat m = arma::sum(gy % (input.each_col() - mean), 1); g = arma::repmat(m, 1, input.n_cols) % -(input.each_col() - mean); - g.each_col() %= 1.0/(variance + eps); - g += (n * gy - arma::repmat((arma::sum(gy, 1)), 1, input.n_cols)); - g.each_col() %= ((1.0 / n) * gamma); - g.each_col() %= (1.0/arma::sqrt(variance + eps)); + g.each_col() %= 1.0 / (variance + eps); + g += (input.n_cols * gy - arma::repmat((arma::sum(gy, 1)), 1, input.n_cols)); + g.each_col() %= ((1.0 / input.n_cols) * gamma); + g.each_col() %= (1.0 / arma::sqrt(variance + eps)); } template @@ -106,11 +110,11 @@ void BatchNorm::Gradient( gradient.reshape(size + size, 1); normalized = input.each_col() - arma::mean(input, 1) - / arma::sqrt(arma::var(input, 1, 1) + eps); + / arma::sqrt(arma::var(input, 1, 1) + eps); gradient.submat(0, 0, gamma.n_elem - 1, 0) = arma::sum(normalized % error, 1); gradient.submat(gamma.n_elem, 0, gradient.n_elem - 1, 0) = - arma::sum(error, 1); + arma::sum(error, 1); } template diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index c1009ccfdb..39cd3d0cf8 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -12,6 +12,10 @@ #ifndef MLPACK_METHODS_ANN_LAYER_LAYER_TYPES_HPP #define MLPACK_METHODS_ANN_LAYER_LAYER_TYPES_HPP +// Increase +#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS +#define BOOST_MPL_LIMIT_LIST_SIZE 50 + #include // Layer modules. @@ -112,7 +116,7 @@ using LayerTypes = boost::variant< BaseLayer*, BaseLayer*, BatchNorm*, -// BilinearInterpolation*, + BilinearInterpolation*, Concat*, ConcatPerformance, arma::mat, arma::mat>*, diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 1dfd1cf680..9e822d261d 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -1330,9 +1330,7 @@ BOOST_AUTO_TEST_CASE(BatchNormTest) << 4.9 << 3.0 << 1.4 << arma::endr << 4.7 << 3.2 << 1.3 << arma::endr; - size_t numUnits = input.n_rows; - - BatchNorm<> model(numUnits); + BatchNorm<> model(input.n_rows); model.Reset(); // Non-Deteministic Forward Pass Test. @@ -1414,7 +1412,7 @@ BOOST_AUTO_TEST_CASE(GradientBatchNormLayerTest) target.ones(1, 256); model = new FFN, NguyenWidrowInitialization>( - input, target); + input, target); model->Add >(); model->Add >(10); model->Add >(10, 2); From 40a289bcd71862ff986f5abc8c8d6746ec53d34a Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Fri, 9 Mar 2018 19:45:06 +0530 Subject: [PATCH 66/79] Increase boost:variant size --- src/mlpack/methods/ann/layer/layer_types.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 39cd3d0cf8..48e8eed0d5 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -12,7 +12,7 @@ #ifndef MLPACK_METHODS_ANN_LAYER_LAYER_TYPES_HPP #define MLPACK_METHODS_ANN_LAYER_LAYER_TYPES_HPP -// Increase +// Increase boost:variant size #define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS #define BOOST_MPL_LIMIT_LIST_SIZE 50 From 37603c1f82e17a2e2c5f65cef4a823bab9e8cf80 Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Fri, 9 Mar 2018 19:49:07 +0530 Subject: [PATCH 67/79] Fix style issue --- src/mlpack/methods/ann/layer/batch_norm_impl.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index e96c34095b..10d1cea45c 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -85,7 +85,6 @@ template void BatchNorm::Backward( const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) { - g.reshape(input.n_rows, input.n_cols); mean = arma::mean(input, 1); From f46b131455493d01a098d0c74bbcf48e870f0296 Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Fri, 9 Mar 2018 19:54:27 +0530 Subject: [PATCH 68/79] Minor style update --- src/mlpack/methods/ann/layer/batch_norm_impl.hpp | 2 +- src/mlpack/tests/ann_layer_test.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index 10d1cea45c..032681120e 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -83,7 +83,7 @@ void BatchNorm::Forward( template template void BatchNorm::Backward( - const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) { g.reshape(input.n_rows, input.n_cols); diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 9e822d261d..04552fe775 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -1353,7 +1353,7 @@ BOOST_AUTO_TEST_CASE(BatchNormTest) model.Backward(std::move(input), std::move(gy), std::move(output)); result << -0.0780 << 0.1376 << -0.0596 << arma::endr << 0.0602 << -0.1317 << 0.0715 << arma::endr - << 0.0835 << -0.1493 << 0.0658 << arma::endr; + << 0.0835 << -0.1493 << 0.0658 << arma::endr; CheckMatrices(output, result, 1e-1); result.clear(); From c088c7b089b35a49673e48decc92c3e03b896fee Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Fri, 9 Mar 2018 20:54:37 +0530 Subject: [PATCH 69/79] Update the maximum size in prereqs.hpp --- src/mlpack/methods/ann/layer/layer_types.hpp | 4 ---- src/mlpack/prereqs.hpp | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 48e8eed0d5..9d609effdd 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -12,10 +12,6 @@ #ifndef MLPACK_METHODS_ANN_LAYER_LAYER_TYPES_HPP #define MLPACK_METHODS_ANN_LAYER_LAYER_TYPES_HPP -// Increase boost:variant size -#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS -#define BOOST_MPL_LIMIT_LIST_SIZE 50 - #include // Layer modules. diff --git a/src/mlpack/prereqs.hpp b/src/mlpack/prereqs.hpp index d373378cf9..a8e0a70809 100644 --- a/src/mlpack/prereqs.hpp +++ b/src/mlpack/prereqs.hpp @@ -65,7 +65,7 @@ using enable_if_t = typename enable_if::type; #undef BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS #undef BOOST_MPL_LIMIT_LIST_SIZE #define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS -#define BOOST_MPL_LIMIT_LIST_SIZE 40 +#define BOOST_MPL_LIMIT_LIST_SIZE 50 // We'll need the necessary boost::serialization features, as well as what we // use with mlpack. In Boost 1.59 and newer, the BOOST_PFTO code is no longer From f72164b527063d1d0cd0677679c849f035fce597 Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Sat, 10 Mar 2018 00:56:22 +0530 Subject: [PATCH 70/79] Fix style issues --- .../methods/ann/layer/batch_norm_impl.hpp | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index 032681120e..61c2060d16 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -21,20 +21,20 @@ namespace mlpack { namespace ann { /** Artificial Neural Network. */ template -BatchNorm::BatchNorm(): - size(10), - eps(1e-7), - deterministic(false) +BatchNorm::BatchNorm() : + size(10), + eps(1e-7), + deterministic(false) { // Nothing to do here. } template BatchNorm::BatchNorm( - const size_t size, const double eps): - size(size), - eps(eps), - deterministic(false) + const size_t size, const double eps) : + size(size), + eps(eps), + deterministic(false) { weights.set_size(size + size, 1); } @@ -101,9 +101,9 @@ void BatchNorm::Backward( template template void BatchNorm::Gradient( - const arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient) + const arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& gradient) { arma::mat normalized(input.n_rows, input.n_cols); gradient.reshape(size + size, 1); @@ -119,7 +119,7 @@ void BatchNorm::Gradient( template template void BatchNorm::serialize( - Archive& ar, const unsigned int /* version */) + Archive& ar, const unsigned int /* version */) { ar & BOOST_SERIALIZATION_NVP(gamma); ar & BOOST_SERIALIZATION_NVP(beta); From e6e8070e1b2dfff9d69fc18aaf7a8b34d1a9b6b2 Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Wed, 14 Mar 2018 10:11:23 +0530 Subject: [PATCH 71/79] Update backward formula --- src/mlpack/methods/ann/layer/batch_norm_impl.hpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index 61c2060d16..42a4cf7d40 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -76,7 +76,7 @@ void BatchNorm::Forward( } output = input.each_col() - mean; - output.each_col() %= gamma / arma::sqrt(variance+eps); + output.each_col() %= gamma / arma::sqrt(variance + eps); output.each_col() += beta; } @@ -85,15 +85,14 @@ template void BatchNorm::Backward( const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) { - g.reshape(input.n_rows, input.n_cols); - mean = arma::mean(input, 1); variance = arma::var(input, 1, 1); arma::mat m = arma::sum(gy % (input.each_col() - mean), 1); - g = arma::repmat(m, 1, input.n_cols) % -(input.each_col() - mean); + g = arma::repmat(m, 1, input.n_cols) % (-input.each_col() + mean); g.each_col() %= 1.0 / (variance + eps); - g += (input.n_cols * gy - arma::repmat((arma::sum(gy, 1)), 1, input.n_cols)); + g += (gy.each_col() - arma::sum(gy, 1)); + g += (input.n_cols - 1) * gy; g.each_col() %= ((1.0 / input.n_cols) * gamma); g.each_col() %= (1.0 / arma::sqrt(variance + eps)); } @@ -105,10 +104,9 @@ void BatchNorm::Gradient( arma::Mat&& error, arma::Mat&& gradient) { - arma::mat normalized(input.n_rows, input.n_cols); - gradient.reshape(size + size, 1); + gradient.set_size(size + size, 1); - normalized = input.each_col() - arma::mean(input, 1) + arma::mat normalized = input.each_col() - arma::mean(input, 1) / arma::sqrt(arma::var(input, 1, 1) + eps); gradient.submat(0, 0, gamma.n_elem - 1, 0) = arma::sum(normalized % error, 1); From f598498559bbdd59b8039cd56725a5054b3a2cc2 Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Wed, 14 Mar 2018 14:31:27 +0530 Subject: [PATCH 72/79] Minor fix --- src/mlpack/methods/ann/layer/batch_norm_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index 42a4cf7d40..504e66ee84 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -89,7 +89,7 @@ void BatchNorm::Backward( variance = arma::var(input, 1, 1); arma::mat m = arma::sum(gy % (input.each_col() - mean), 1); - g = arma::repmat(m, 1, input.n_cols) % (-input.each_col() + mean); + g = arma::repmat(m, 1, input.n_cols) % (mean - input.each_col()); g.each_col() %= 1.0 / (variance + eps); g += (gy.each_col() - arma::sum(gy, 1)); g += (input.n_cols - 1) * gy; From 5fa2465ee659fc58571d05a4f20cbee4b1161514 Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Wed, 14 Mar 2018 21:28:07 +0530 Subject: [PATCH 73/79] Removed use of repmat --- src/mlpack/methods/ann/layer/batch_norm_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index 504e66ee84..d8f0e2c90f 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -89,8 +89,8 @@ void BatchNorm::Backward( variance = arma::var(input, 1, 1); arma::mat m = arma::sum(gy % (input.each_col() - mean), 1); - g = arma::repmat(m, 1, input.n_cols) % (mean - input.each_col()); - g.each_col() %= 1.0 / (variance + eps); + g = (mean - input.each_col()); + g.each_col() %= m; g += (gy.each_col() - arma::sum(gy, 1)); g += (input.n_cols - 1) * gy; g.each_col() %= ((1.0 / input.n_cols) * gamma); From c471518ccb1b470536e57595b0c85b4105951fbf Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Wed, 14 Mar 2018 22:43:13 +0530 Subject: [PATCH 74/79] Fix the formula --- src/mlpack/methods/ann/layer/batch_norm_impl.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index d8f0e2c90f..9d5dd4bfd6 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -91,6 +91,7 @@ void BatchNorm::Backward( arma::mat m = arma::sum(gy % (input.each_col() - mean), 1); g = (mean - input.each_col()); g.each_col() %= m; + g.each_col() %= 1.0/(variance + eps); g += (gy.each_col() - arma::sum(gy, 1)); g += (input.n_cols - 1) * gy; g.each_col() %= ((1.0 / input.n_cols) * gamma); From 0cab3a987b19ae76effd3e4af8cc538b36c87240 Mon Sep 17 00:00:00 2001 From: manthan-r-sheth Date: Fri, 16 Mar 2018 19:54:16 +0530 Subject: [PATCH 75/79] Add name to contributors list --- COPYRIGHT.txt | 1 + src/mlpack/core.hpp | 1 + 2 files changed, 2 insertions(+) diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index 2cdaa62c00..e02c52d769 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -97,6 +97,7 @@ Copyright: Copyright 2018, Prabhat Sharma Copyright 2018, Tan Jun An Copyright 2018, Moksh Jain + Copyright 2018, Manthan-R-Sheth License: BSD-3-clause All rights reserved. diff --git a/src/mlpack/core.hpp b/src/mlpack/core.hpp index d9a8b636ca..6a0d6b7e03 100644 --- a/src/mlpack/core.hpp +++ b/src/mlpack/core.hpp @@ -240,6 +240,7 @@ * - Prabhat Sharma * - Tan Jun An * - Moksh Jain + * - Manthan-R-Sheth */ // First, include all of the prerequisites. From c9c5acb9e0a4648f8f06c24cb821adc6005323f7 Mon Sep 17 00:00:00 2001 From: Wenhao Huang Date: Sat, 17 Mar 2018 16:26:07 +0800 Subject: [PATCH 76/79] change func name and add comments --- src/mlpack/core/math/random.hpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/mlpack/core/math/random.hpp b/src/mlpack/core/math/random.hpp index 8e90d9bcb8..4e1425466b 100644 --- a/src/mlpack/core/math/random.hpp +++ b/src/mlpack/core/math/random.hpp @@ -50,9 +50,13 @@ inline void RandomSeed(const size_t seed) /** * Set the random seed to a fixed number. + * This function is used in binding tests to set a fixed random seed before + * calling mlpack(). In this way we can test whether a certain parameter makes + * a difference to execution of CLI binding. + * Refer to pull request #1306 for discussion on this function. */ #if (BINDING_TYPE == BINDING_TYPE_TEST) -inline void SetFixedRandomSeed() +inline void FixedRandomSeed() { const static size_t seed = rand(); randGen.seed((uint32_t) seed); From 7dbbbb43daf7d9d078fe5c9632c473897b1ca5ca Mon Sep 17 00:00:00 2001 From: akhandait Date: Sat, 17 Mar 2018 19:59:39 +0530 Subject: [PATCH 77/79] change link to tutorial page --- doc/tutorials/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/README.md b/doc/tutorials/README.md index 0e68acd6a8..137136a680 100644 --- a/doc/tutorials/README.md +++ b/doc/tutorials/README.md @@ -1,7 +1,7 @@ ## Tutorials -Tutorials for mlpack can be found [here : mlpack tutorials](http://www.mlpack.org/tutorials.html). +Tutorials for mlpack can be found [here : mlpack tutorials](http://www.mlpack.org/docs/mlpack-git/doxygen/tutorials.html). ### General mlpack tutorials From 0f023d7395b75ad430bb9e06dd9d5687031e7e7f Mon Sep 17 00:00:00 2001 From: akhandait Date: Sun, 18 Mar 2018 10:40:38 +0530 Subject: [PATCH 78/79] use https instead of http --- doc/tutorials/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/README.md b/doc/tutorials/README.md index 137136a680..500f1f31ee 100644 --- a/doc/tutorials/README.md +++ b/doc/tutorials/README.md @@ -1,7 +1,7 @@ ## Tutorials -Tutorials for mlpack can be found [here : mlpack tutorials](http://www.mlpack.org/docs/mlpack-git/doxygen/tutorials.html). +Tutorials for mlpack can be found [here : mlpack tutorials](https://www.mlpack.org/docs/mlpack-git/doxygen/tutorials.html). ### General mlpack tutorials From 49e3cda97d5d0244c1e2d60c14933dc3ff52fe5f Mon Sep 17 00:00:00 2001 From: Shikhar Jaiswal Date: Sat, 17 Mar 2018 14:35:00 +0530 Subject: [PATCH 79/79] Improvements and Speedups --- .../optimizers/sdp/lrsdp_function_impl.hpp | 3 +- .../ann/convolution_rules/fft_convolution.hpp | 41 +++++++++---------- .../convolution_rules/naive_convolution.hpp | 10 ++--- .../ann/convolution_rules/svd_convolution.hpp | 15 +++---- .../methods/ann/layer/convolution_impl.hpp | 14 +++---- .../decision_tree/decision_tree_impl.hpp | 2 +- src/mlpack/tests/convolution_test.cpp | 20 ++++----- 7 files changed, 49 insertions(+), 56 deletions(-) diff --git a/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp b/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp index 978c7da33f..f27c3d458c 100644 --- a/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp +++ b/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp @@ -51,7 +51,8 @@ LRSDPFunction::LRSDPFunction(const size_t numSparseConstraints, } template -double LRSDPFunction::Evaluate(const arma::mat& coordinates) const +double LRSDPFunction::Evaluate(const arma::mat& /* coordinates */) + const { // Note: We don't require to update the R*R^T matrix here as the current // function is only used by AugLagrangian, which do not update the coordinates diff --git a/src/mlpack/methods/ann/convolution_rules/fft_convolution.hpp b/src/mlpack/methods/ann/convolution_rules/fft_convolution.hpp index f573ce9ca2..b362d3ecce 100644 --- a/src/mlpack/methods/ann/convolution_rules/fft_convolution.hpp +++ b/src/mlpack/methods/ann/convolution_rules/fft_convolution.hpp @@ -21,8 +21,8 @@ namespace ann /** Artificial Neural Network. */ { /** * Computes the two-dimensional convolution through fft. This class allows - * specification of the type of the border type. The convolution can be compute - * with the valid border type of the full border type (default). + * specification of the type of the border type. The convolution can be + * computed with the valid border type of the full border type (default). * * FullConvolution: returns the full two-dimensional convolution. * ValidConvolution: returns only those parts of the convolution that are @@ -40,12 +40,12 @@ class FFTConvolution /* * Perform a convolution through fft (valid mode). This method only supports * input which is even on the last dimension. In case of an odd input width, a - * user can manually pad the imput or specify the padLastDim parameter which + * user can manually pad the input or specify the padLastDim parameter which * takes care of the padding. The filter instead can have any size. When using - * the valid mode the filters has to be smaller than the input. + * the valid mode the filter has to be smaller than the input. * * @param input Input used to perform the convolution. - * @param filter Filter used to perform the conolution. + * @param filter Filter used to perform the convolution. * @param output Output data that contains the results of the convolution. */ template @@ -64,23 +64,23 @@ class FFTConvolution // Pad filter and input to the output shape. filterPadded.resize(inputPadded.n_rows, inputPadded.n_cols); - output = arma::real(ifft2(arma::fft2(inputPadded) % arma::fft2( + arma::Mat temp = arma::real(ifft2(arma::fft2(inputPadded) % arma::fft2( filterPadded))); // Extract the region of interest. We don't need to handle the padLastDim in // a special way we just cut it out from the output matrix. - output = output.submat(filter.n_rows - 1, filter.n_cols - 1, + output = temp.submat(filter.n_rows - 1, filter.n_cols - 1, input.n_rows - 1, input.n_cols - 1); } /* * Perform a convolution through fft (full mode). This method only supports * input which is even on the last dimension. In case of an odd input width, a - * user can manually pad the imput or specify the padLastDim parameter which + * user can manually pad the input or specify the padLastDim parameter which * takes care of the padding. The filter instead can have any size. * * @param input Input used to perform the convolution. - * @param filter Filter used to perform the conolution. + * @param filter Filter used to perform the convolution. * @param output Output data that contains the results of the convolution. */ template @@ -110,12 +110,12 @@ class FFTConvolution filterPadded.resize(outputRows, outputCols); // Perform FFT and IFFT - output = arma::real(ifft2(arma::fft2(inputPadded) % arma::fft2( + arma::Mat temp = arma::real(ifft2(arma::fft2(inputPadded) % arma::fft2( filterPadded))); // Extract the region of interest. We don't need to handle the padLastDim // parameter in a special way we just cut it out from the output matrix. - output = output.submat(filter.n_rows - 1, filter.n_cols - 1, + output = temp.submat(filter.n_rows - 1, filter.n_cols - 1, 2 * (filter.n_rows - 1) + input.n_rows - 1, 2 * (filter.n_cols - 1) + input.n_cols - 1); } @@ -123,12 +123,12 @@ class FFTConvolution /* * Perform a convolution through fft using 3rd order tensors. This method only * supports input which is even on the last dimension. In case of an odd input - * width, a user can manually pad the imput or specify the padLastDim + * width, a user can manually pad the input or specify the padLastDim * parameter which takes care of the padding. The filter instead can have any * size. * * @param input Input used to perform the convolution. - * @param filter Filter used to perform the conolution. + * @param filter Filter used to perform the convolution. * @param output Output data that contains the results of the convolution. */ template @@ -147,8 +147,7 @@ class FFTConvolution for (size_t i = 1; i < input.n_slices; i++) { FFTConvolution::Convolution(input.slice(i), filter.slice(i), - convOutput); - output.slice(i) = convOutput; + output.slice(i)); } } @@ -156,11 +155,11 @@ class FFTConvolution * Perform a convolution through fft using dense matrix as input and a 3rd * order tensors as filter and output. This method only supports input which * is even on the last dimension. In case of an odd input width, a user can - * manually pad the imput or specify the padLastDim parameter which takes care + * manually pad the input or specify the padLastDim parameter which takes care * of the padding. The filter instead can have any size. * * @param input Input used to perform the convolution. - * @param filter Filter used to perform the conolution. + * @param filter Filter used to perform the convolution. * @param output Output data that contains the results of the convolution. */ template @@ -179,8 +178,7 @@ class FFTConvolution for (size_t i = 1; i < filter.n_slices; i++) { FFTConvolution::Convolution(input, filter.slice(i), - convOutput); - output.slice(i) = convOutput; + output.slice(i)); } } @@ -189,7 +187,7 @@ class FFTConvolution * dense matrix as filter. * * @param input Input used to perform the convolution. - * @param filter Filter used to perform the conolution. + * @param filter Filter used to perform the convolution. * @param output Output data that contains the results of the convolution. */ template @@ -208,8 +206,7 @@ class FFTConvolution for (size_t i = 1; i < input.n_slices; i++) { FFTConvolution::Convolution(input.slice(i), filter, - convOutput); - output.slice(i) = convOutput; + output.slice(i)); } } }; // class FFTConvolution diff --git a/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp b/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp index c27225f127..90882c3b7d 100644 --- a/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp +++ b/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp @@ -39,7 +39,7 @@ class NaiveConvolution * Perform a convolution (valid mode). * * @param input Input used to perform the convolution. - * @param filter Filter used to perform the conolution. + * @param filter Filter used to perform the convolution. * @param output Output data that contains the results of the convolution. * @param dW Stride of filter application in the x direction. * @param dH Stride of filter application in the y direction. @@ -79,7 +79,7 @@ class NaiveConvolution * Perform a convolution (full mode). * * @param input Input used to perform the convolution. - * @param filter Filter used to perform the conolution. + * @param filter Filter used to perform the convolution. * @param output Output data that contains the results of the convolution. * @param dW Stride of filter application in the x direction. * @param dH Stride of filter application in the y direction. @@ -111,7 +111,7 @@ class NaiveConvolution * Perform a convolution using 3rd order tensors. * * @param input Input used to perform the convolution. - * @param filter Filter used to perform the conolution. + * @param filter Filter used to perform the convolution. * @param output Output data that contains the results of the convolution. * @param dW Stride of filter application in the x direction. * @param dH Stride of filter application in the y direction. @@ -143,7 +143,7 @@ class NaiveConvolution * as filter and output. * * @param input Input used to perform the convolution. - * @param filter Filter used to perform the conolution. + * @param filter Filter used to perform the convolution. * @param output Output data that contains the results of the convolution. * @param dW Stride of filter application in the x direction. * @param dH Stride of filter application in the y direction. @@ -175,7 +175,7 @@ class NaiveConvolution * dense matrix as filter. * * @param input Input used to perform the convolution. - * @param filter Filter used to perform the conolution. + * @param filter Filter used to perform the convolution. * @param output Output data that contains the results of the convolution. * @param dW Stride of filter application in the x direction. * @param dH Stride of filter application in the y direction. diff --git a/src/mlpack/methods/ann/convolution_rules/svd_convolution.hpp b/src/mlpack/methods/ann/convolution_rules/svd_convolution.hpp index 7cd50470ab..8ed0a9c415 100644 --- a/src/mlpack/methods/ann/convolution_rules/svd_convolution.hpp +++ b/src/mlpack/methods/ann/convolution_rules/svd_convolution.hpp @@ -3,7 +3,7 @@ * @author Marcus Edel * * Implementation of the convolution using the singular value decomposition to - * speeded up the computation. + * speed up the computation. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -24,7 +24,7 @@ namespace ann /** Artificial Neural Network. */ { /** * Computes the two-dimensional convolution using singular value decomposition. * This class allows specification of the type of the border type. The - * convolution can be compute with the valid border type of the full border + * convolution can be computed with the valid border type of the full border * type (default). * * FullConvolution: returns the full two-dimensional convolution. @@ -87,13 +87,13 @@ class SVDConvolution NaiveConvolution::Convolution(subOutput, U.unsafe_col(0), output); + arma::Mat temp; for (size_t r = 1; r < rank; r++) { subFilter = V.unsafe_col(r) * s(r); NaiveConvolution::Convolution(input, subFilter, subOutput); - arma::Mat temp; subOutput = subOutput.t(); NaiveConvolution::Convolution(subOutput, U.unsafe_col(r), temp); @@ -134,8 +134,7 @@ class SVDConvolution for (size_t i = 1; i < input.n_slices; i++) { SVDConvolution::Convolution(input.slice(i), filter.slice(i), - convOutput); - output.slice(i) = convOutput; + output.slice(i)); } } @@ -164,8 +163,7 @@ class SVDConvolution for (size_t i = 1; i < filter.n_slices; i++) { SVDConvolution::Convolution(input, filter.slice(i), - convOutput); - output.slice(i) = convOutput; + output.slice(i)); } } @@ -194,8 +192,7 @@ class SVDConvolution for (size_t i = 1; i < input.n_slices; i++) { SVDConvolution::Convolution(input.slice(i), filter, - convOutput); - output.slice(i) = convOutput; + output.slice(i)); } } }; // class SVDConvolution diff --git a/src/mlpack/methods/ann/layer/convolution_impl.hpp b/src/mlpack/methods/ann/layer/convolution_impl.hpp index 3b01406d1b..48da43af38 100644 --- a/src/mlpack/methods/ann/layer/convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/convolution_impl.hpp @@ -171,8 +171,8 @@ void Convolution< >::Backward( const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { - arma::cube mappedError = arma::cube(gy.memptr(), - outputWidth, outputHeight, outSize); + arma::cube mappedError(gy.memptr(), outputWidth, outputHeight, outSize, + false, false); gTemp = arma::zeros >(inputTemp.n_rows, inputTemp.n_cols, inputTemp.n_slices); @@ -265,12 +265,10 @@ void Convolution< { for (size_t i = 0; i < output.n_slices; i++) { - arma::mat subOutput = output.slice(i); - - gradientTemp.slice(s) += subOutput.submat(subOutput.n_rows / 2, - subOutput.n_cols / 2, - subOutput.n_rows / 2 + gradientTemp.n_rows - 1, - subOutput.n_cols / 2 + gradientTemp.n_cols - 1); + gradientTemp.slice(s) += output.slice(i).submat(output.n_rows / 2, + output.n_cols / 2, + output.n_rows / 2 + gradientTemp.n_rows - 1, + output.n_cols / 2 + gradientTemp.n_cols - 1); } } else diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index 9e9a971c4e..91a0827982 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -414,7 +414,7 @@ void DecisionTree(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, weights, - minimumLeafSize); + minimumLeafSize, minimumGainSplit); } //! Train on the given weighted data. diff --git a/src/mlpack/tests/convolution_test.cpp b/src/mlpack/tests/convolution_test.cpp index a277b9cb41..739a7cae40 100644 --- a/src/mlpack/tests/convolution_test.cpp +++ b/src/mlpack/tests/convolution_test.cpp @@ -29,7 +29,7 @@ BOOST_AUTO_TEST_SUITE(ConvolutionTest); * Implementation of the convolution function test. * * @param input Input used to perform the convolution. - * @param filter Filter used to perform the conolution. + * @param filter Filter used to perform the convolution. * @param output The reference output data that contains the results of the * convolution. * @@ -43,7 +43,7 @@ void Convolution2DMethodTest(const arma::mat input, arma::mat convOutput; ConvolutionFunction::Convolution(input, filter, convOutput); - // Check the outut dimension. + // Check the output dimension. bool b = (convOutput.n_rows == output.n_rows) && (convOutput.n_cols == output.n_cols); BOOST_REQUIRE_EQUAL(b, 1); @@ -59,7 +59,7 @@ void Convolution2DMethodTest(const arma::mat input, * Implementation of the convolution function test using 3rd order tensors. * * @param input Input used to perform the convolution. - * @param filter Filter used to perform the conolution. + * @param filter Filter used to perform the convolution. * @param output The reference output data that contains the results of the * convolution. * @@ -91,7 +91,7 @@ void Convolution3DMethodTest(const arma::cube input, * and a 3rd order tensors as filter and output (batch modus). * * @param input Input used to perform the convolution. - * @param filter Filter used to perform the conolution. + * @param filter Filter used to perform the convolution. * @param output The reference output data that contains the results of the * convolution. * @@ -146,7 +146,7 @@ BOOST_AUTO_TEST_CASE(ValidConvolution2DTest) output); // Perform the convolution using singular value decomposition to - // speeded up the computation. + // speed up the computation. Convolution2DMethodTest >(input, filter, output); } @@ -183,7 +183,7 @@ BOOST_AUTO_TEST_CASE(FullConvolution2DTest) output); // Perform the convolution using singular value decomposition to - // speeded up the computation. + // speed up the computation. Convolution2DMethodTest >(input, filter, output); } @@ -228,7 +228,7 @@ BOOST_AUTO_TEST_CASE(ValidConvolution3DTest) filterCube, outputCube); // Perform the convolution using using the singular value decomposition to - // speeded up the computation. + // speed up the computation. Convolution3DMethodTest >(inputCube, filterCube, outputCube); } @@ -277,7 +277,7 @@ BOOST_AUTO_TEST_CASE(FullConvolution3DTest) filterCube, outputCube); // Perform the convolution using using the singular value decomposition to - // speeded up the computation. + // speed up the computation. Convolution3DMethodTest >(inputCube, filterCube, outputCube); } @@ -319,7 +319,7 @@ BOOST_AUTO_TEST_CASE(ValidConvolutionBatchTest) filterCube, outputCube); // Perform the convolution using using the singular value decomposition to - // speeded up the computation. + // speed up the computation. ConvolutionMethodBatchTest >(input, filterCube, outputCube); } @@ -365,7 +365,7 @@ BOOST_AUTO_TEST_CASE(FullConvolutionBatchTest) filterCube, outputCube); // Perform the convolution using using the singular value decomposition to - // speeded up the computation. + // speed up the computation. ConvolutionMethodBatchTest >(input, filterCube, outputCube); }