From b2c4bcfb174e4d4ee2bcf21f5be5784e52e05f1c Mon Sep 17 00:00:00 2001 From: dakshitagrawal97 Date: Tue, 27 Feb 2018 01:19:05 +0530 Subject: [PATCH 01/15] Add He initialization rule --- src/mlpack/methods/ann/init_rules/he_init.hpp | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 src/mlpack/methods/ann/init_rules/he_init.hpp diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp new file mode 100644 index 0000000000..58e9732b3b --- /dev/null +++ b/src/mlpack/methods/ann/init_rules/he_init.hpp @@ -0,0 +1,100 @@ +/** + * @file he_init.hpp + * @author Dakshit Agrawal + * + * Intialization rule given by He et. al. for neural networks. The He + * initialization initializes weights of the neural network to better + * suit the rectified activation units. + * + * For more information, the following paper can be referred to: + * + * @code + * @article{He2015DelvingDI, + * title={Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification}, + * author={Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun}, + * journal={2015 IEEE International Conference on Computer Vision (ICCV)}, + * year={2015}, + * pages={1026-1034}} + * @endcode + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_METHODS_ANN_INIT_RULES_HE_INIT_HPP +#define MLPACK_METHODS_ANN_INIT_RULES_HE_INIT_HPP + +#include +#include + +using namespace mlpack::math; + +namespace mlpack { + namespace ann /** Artificial Neural Network. */ { + +/** + * This class is used to initialize weight matrix with the He initialization rule. + */ + class HeInitialization + { + public: + /** + * Initialize the HeInitialization object. + * + */ + HeInitialization() + { + // Nothing to do here. + } + + /** + * Initialize the elements of the weight matrix with the He initialization + * rule. + * + * @param W Weight matrix to initialize. + * @param rows Number of rows. + * @param cols Number of columns. + */ + void Initialize(arma::mat& W, + const size_t rows, + const size_t cols) + { + double_t variance = 2 / rows; + + if (W.is_empty()) + { + W = arma::mat(rows, cols); + } + + W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } ); + } + + /** + * Initialize the elements of the specified weight 3rd order tensor + * with He initialization rule. + * + * @param W Weight matrix to initialize. + * @param rows Number of rows. + * @param cols Number of columns. + * @param slice Numbers of slices. + */ + void Initialize(arma::cube & W, + const size_t rows, + const size_t cols, + const size_t slices) + { + W = arma::cube(rows, cols, slices); + + for (size_t i = 0; i < slices; i++) { + Initialize(W.slice(i), rows, cols); + } + } + + }; // class HeInitialization + + } // namespace ann +} // namespace mlpack + +#endif \ No newline at end of file From 0f2ff46f7344e309701895123e14469e0bdb514f Mon Sep 17 00:00:00 2001 From: dakshitagrawal97 Date: Tue, 27 Feb 2018 01:20:20 +0530 Subject: [PATCH 02/15] Add Lecun Normal initialization rule. --- .../ann/init_rules/lecun_normal_init.hpp | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp diff --git a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp new file mode 100644 index 0000000000..14cf63d07d --- /dev/null +++ b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp @@ -0,0 +1,105 @@ +/** + * @file lecun_normal_init.hpp + * @author Dakshit Agrawal + * + * Intialization rule given by Lecun et. al. for neural networks and + * also mentioned in Self Normalizing Networks. + * + * For more information, the following papers can be referred to: + * + * @code + * @inproceedings{conf/nips/KlambauerUMH17, + * title = {Self-Normalizing Neural Networks.}, + * author = {Klambauer, Günter and Unterthiner, Thomas and Mayr, Andreas and Hochreiter, Sepp}, + * pages = {972-981}, + * year = 2017} + * + * @inproceedings{LeCun:1998:EB:645754.668382, + * title = {Efficient BackProp}, + * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B. and M\"{u}ller, Klaus-Robert}, + * year = {1998}, + * pages = {9--50}} + * @endcode + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_METHODS_ANN_INIT_RULES_LECUN_NORMAL_INIT_HPP +#define MLPACK_METHODS_ANN_INIT_RULES_LECUN_NORMAL_INIT_HPP + +#include +#include + +using namespace mlpack::math; + +namespace mlpack { + namespace ann /** Artificial Neural Network. */ { + +/** + * This class is used to initialize weight matrix with the Lecun Normalization + * initialization rule. + */ + class LecunNormalInitialization + { + public: + /** + * Initialize the LecunNormalInitialization object. + * + */ + LecunNormalInitialization() + { + // Nothing to do here. + } + + /** + * Initialize the elements of the weight matrix with the Lecun + * Normal initialization rule. + * + * @param W Weight matrix to initialize. + * @param rows Number of rows. + * @param cols Number of columns. + */ + void Initialize(arma::mat& W, + const size_t rows, + const size_t cols) + { + double_t variance = 1 / rows; + + if (W.is_empty()) + { + W = arma::mat(rows, cols); + } + + W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } ); + } + + /** + * Initialize the elements of the specified weight 3rd order tensor + * with Lecun Normal initialization rule. + * + * @param W Weight matrix to initialize. + * @param rows Number of rows. + * @param cols Number of columns. + * @param slice Numbers of slices. + */ + void Initialize(arma::cube & W, + const size_t rows, + const size_t cols, + const size_t slices) + { + W = arma::cube(rows, cols, slices); + + for (size_t i = 0; i < slices; i++) { + Initialize(W.slice(i), rows, cols); + } + } + + }; // class LecunNormalInitialization + + } // namespace ann +} // namespace mlpack + +#endif \ No newline at end of file From 8f471a86cd3720ca90e6d7e5a780886b1c2578ae Mon Sep 17 00:00:00 2001 From: dakshitagrawal97 Date: Tue, 27 Feb 2018 01:22:53 +0530 Subject: [PATCH 03/15] update CMakeLists --- src/mlpack/methods/ann/init_rules/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mlpack/methods/ann/init_rules/CMakeLists.txt b/src/mlpack/methods/ann/init_rules/CMakeLists.txt index e8172df613..83531f9191 100644 --- a/src/mlpack/methods/ann/init_rules/CMakeLists.txt +++ b/src/mlpack/methods/ann/init_rules/CMakeLists.txt @@ -3,8 +3,10 @@ set(SOURCES const_init.hpp gaussian_init.hpp + he_init.hpp init_rules_traits.hpp kathirvalavakumar_subavathi_init.hpp + lecun_normal_init.hpp network_init.hpp nguyen_widrow_init.hpp oivs_init.hpp From f29ae55bb1e44d315afe1be3da0a3a4069467b10 Mon Sep 17 00:00:00 2001 From: dakshitagrawal97 Date: Wed, 28 Feb 2018 22:26:51 +0530 Subject: [PATCH 04/15] add He initialization test --- src/mlpack/tests/init_rules_test.cpp | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/mlpack/tests/init_rules_test.cpp b/src/mlpack/tests/init_rules_test.cpp index 079c4263e5..c6d1d05936 100644 --- a/src/mlpack/tests/init_rules_test.cpp +++ b/src/mlpack/tests/init_rules_test.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include "test_tools.hpp" @@ -197,6 +198,32 @@ BOOST_AUTO_TEST_CASE(GaussianInitTest) BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices); } +/** +* Simple test of the HeInitialization class. +*/ +BOOST_AUTO_TEST_CASE(HeInitTest) +{ + const size_t rows = 4; + const size_t cols = 4; + const size_t slices = 2; + + arma::mat weights; + arma::cube weights3d; + + HeInitialization initialization; + + initialization.Initialize(weights, rows, cols); + initialization.Initialize(weights3d, rows, cols, slices); + + BOOST_REQUIRE_EQUAL(weights.n_rows, rows); + BOOST_REQUIRE_EQUAL(weights.n_cols, cols); + + BOOST_REQUIRE_EQUAL(weights3d.n_rows, rows); + BOOST_REQUIRE_EQUAL(weights3d.n_cols, cols); + BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices); +} + + /** * Simple test of the NetworkInitialization class, we test it with every * implemented initialization rule and make sure the output is reasonable. From a4f2ed529ebb4a9e2201e4c6661e6db05e28d12a Mon Sep 17 00:00:00 2001 From: dakshitagrawal97 Date: Wed, 28 Feb 2018 22:33:21 +0530 Subject: [PATCH 05/15] add lecun normal initialization tests --- src/mlpack/tests/init_rules_test.cpp | 32 +++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/src/mlpack/tests/init_rules_test.cpp b/src/mlpack/tests/init_rules_test.cpp index c6d1d05936..98dea98c3a 100644 --- a/src/mlpack/tests/init_rules_test.cpp +++ b/src/mlpack/tests/init_rules_test.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include "test_tools.hpp" @@ -210,10 +211,35 @@ BOOST_AUTO_TEST_CASE(HeInitTest) arma::mat weights; arma::cube weights3d; - HeInitialization initialization; + HeInitialization initializer; - initialization.Initialize(weights, rows, cols); - initialization.Initialize(weights3d, rows, cols, slices); + initializer.Initialize(weights, rows, cols); + initializer.Initialize(weights3d, rows, cols, slices); + + BOOST_REQUIRE_EQUAL(weights.n_rows, rows); + BOOST_REQUIRE_EQUAL(weights.n_cols, cols); + + BOOST_REQUIRE_EQUAL(weights3d.n_rows, rows); + BOOST_REQUIRE_EQUAL(weights3d.n_cols, cols); + BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices); +} + +/** +* Simple test of the LecunNormalInitialization class. +*/ +BOOST_AUTO_TEST_CASE(LecunNormalInitTest) +{ + const size_t rows = 4; + const size_t cols = 4; + const size_t slices = 2; + + arma::mat weights; + arma::cube weights3d; + + LecunNormalInitialization initializer; + + initializer.Initialize(weights, rows, cols); + initializer.Initialize(weights3d, rows, cols, slices); BOOST_REQUIRE_EQUAL(weights.n_rows, rows); BOOST_REQUIRE_EQUAL(weights.n_cols, cols); From 84db5a687fe157052511e2459e08ee63bb980149 Mon Sep 17 00:00:00 2001 From: dakshitagrawal97 Date: Wed, 28 Feb 2018 22:39:05 +0530 Subject: [PATCH 06/15] fix style errors --- src/mlpack/methods/ann/init_rules/he_init.hpp | 109 ++++++++--------- .../ann/init_rules/lecun_normal_init.hpp | 115 +++++++++--------- 2 files changed, 111 insertions(+), 113 deletions(-) diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp index 58e9732b3b..7f0f7888aa 100644 --- a/src/mlpack/methods/ann/init_rules/he_init.hpp +++ b/src/mlpack/methods/ann/init_rules/he_init.hpp @@ -32,69 +32,68 @@ using namespace mlpack::math; namespace mlpack { - namespace ann /** Artificial Neural Network. */ { +namespace ann /** Artificial Neural Network. */ { /** * This class is used to initialize weight matrix with the He initialization rule. */ - class HeInitialization +class HeInitialization +{ + public: + /** + * Initialize the HeInitialization object. + * + */ + HeInitialization() + { + // Nothing to do here. + } + + /** + * Initialize the elements of the weight matrix with the He initialization + * rule. + * + * @param W Weight matrix to initialize. + * @param rows Number of rows. + * @param cols Number of columns. + */ + void Initialize(arma::mat& W, + const size_t rows, + const size_t cols) + { + double_t variance = 2 / rows; + + if (W.is_empty()) { - public: - /** - * Initialize the HeInitialization object. - * - */ - HeInitialization() - { - // Nothing to do here. - } + W = arma::mat(rows, cols); + } - /** - * Initialize the elements of the weight matrix with the He initialization - * rule. - * - * @param W Weight matrix to initialize. - * @param rows Number of rows. - * @param cols Number of columns. - */ - void Initialize(arma::mat& W, - const size_t rows, - const size_t cols) - { - double_t variance = 2 / rows; + W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } ); + } - if (W.is_empty()) - { - W = arma::mat(rows, cols); - } + /** + * Initialize the elements of the specified weight 3rd order tensor + * with He initialization rule. + * + * @param W Weight matrix to initialize. + * @param rows Number of rows. + * @param cols Number of columns. + * @param slice Numbers of slices. + */ + void Initialize(arma::cube & W, + const size_t rows, + const size_t cols, + const size_t slices) + { + W = arma::cube(rows, cols, slices); - W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } ); - } + for (size_t i = 0; i < slices; i++) { + Initialize(W.slice(i), rows, cols); + } + } +}; // class HeInitialization - /** - * Initialize the elements of the specified weight 3rd order tensor - * with He initialization rule. - * - * @param W Weight matrix to initialize. - * @param rows Number of rows. - * @param cols Number of columns. - * @param slice Numbers of slices. - */ - void Initialize(arma::cube & W, - const size_t rows, - const size_t cols, - const size_t slices) - { - W = arma::cube(rows, cols, slices); - - for (size_t i = 0; i < slices; i++) { - Initialize(W.slice(i), rows, cols); - } - } - - }; // class HeInitialization - - } // namespace ann +} // namespace ann } // namespace mlpack -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp index 14cf63d07d..92bd9b7ee9 100644 --- a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp +++ b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp @@ -36,70 +36,69 @@ using namespace mlpack::math; namespace mlpack { - namespace ann /** Artificial Neural Network. */ { +namespace ann /** Artificial Neural Network. */ { /** - * This class is used to initialize weight matrix with the Lecun Normalization - * initialization rule. - */ - class LecunNormalInitialization +* This class is used to initialize weight matrix with the Lecun Normalization +* initialization rule. +*/ +class LecunNormalInitialization +{ + public: + /** + * Initialize the LecunNormalInitialization object. + * + */ + LecunNormalInitialization() + { + // Nothing to do here. + } + + /** + * Initialize the elements of the weight matrix with the Lecun + * Normal initialization rule. + * + * @param W Weight matrix to initialize. + * @param rows Number of rows. + * @param cols Number of columns. + */ + void Initialize(arma::mat& W, + const size_t rows, + const size_t cols) + { + double_t variance = 1 / rows; + + if (W.is_empty()) { - public: - /** - * Initialize the LecunNormalInitialization object. - * - */ - LecunNormalInitialization() - { - // Nothing to do here. - } + W = arma::mat(rows, cols); + } - /** - * Initialize the elements of the weight matrix with the Lecun - * Normal initialization rule. - * - * @param W Weight matrix to initialize. - * @param rows Number of rows. - * @param cols Number of columns. - */ - void Initialize(arma::mat& W, - const size_t rows, - const size_t cols) - { - double_t variance = 1 / rows; + W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } ); + } - if (W.is_empty()) - { - W = arma::mat(rows, cols); - } + /** + * Initialize the elements of the specified weight 3rd order tensor + * with Lecun Normal initialization rule. + * + * @param W Weight matrix to initialize. + * @param rows Number of rows. + * @param cols Number of columns. + * @param slice Numbers of slices. + */ + void Initialize(arma::cube & W, + const size_t rows, + const size_t cols, + const size_t slices) + { + W = arma::cube(rows, cols, slices); - W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } ); - } + for (size_t i = 0; i < slices; i++) { + Initialize(W.slice(i), rows, cols); + } + } +}; // class LecunNormalInitialization - /** - * Initialize the elements of the specified weight 3rd order tensor - * with Lecun Normal initialization rule. - * - * @param W Weight matrix to initialize. - * @param rows Number of rows. - * @param cols Number of columns. - * @param slice Numbers of slices. - */ - void Initialize(arma::cube & W, - const size_t rows, - const size_t cols, - const size_t slices) - { - W = arma::cube(rows, cols, slices); - - for (size_t i = 0; i < slices; i++) { - Initialize(W.slice(i), rows, cols); - } - } - - }; // class LecunNormalInitialization - - } // namespace ann +} // namespace ann } // namespace mlpack -#endif \ No newline at end of file +#endif From 546e6d079965dd82961560007e70277a06b27568 Mon Sep 17 00:00:00 2001 From: dakshitagrawal97 Date: Thu, 1 Mar 2018 07:30:13 +0530 Subject: [PATCH 07/15] minor changes for performance improvement and documentation --- src/mlpack/methods/ann/init_rules/he_init.hpp | 14 +++++++++----- .../methods/ann/init_rules/lecun_normal_init.hpp | 14 +++++++++----- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp index 7f0f7888aa..8d569f725b 100644 --- a/src/mlpack/methods/ann/init_rules/he_init.hpp +++ b/src/mlpack/methods/ann/init_rules/he_init.hpp @@ -29,8 +29,6 @@ #include #include -using namespace mlpack::math; - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -61,11 +59,14 @@ class HeInitialization const size_t rows, const size_t cols) { - double_t variance = 2 / rows; + // He initialization rule says to initialize weights with random + // values taken from a gaussian distribution with mean = 0 and + // standard deviation = sqrt(2/rows), i.e. variance = (2/rows). + double_t variance = 2.0 / rows; if (W.is_empty()) { - W = arma::mat(rows, cols); + W.set_size(rows, cols); } W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } ); @@ -85,7 +86,10 @@ class HeInitialization const size_t cols, const size_t slices) { - W = arma::cube(rows, cols, slices); + if (W.is_empty()) + { + W.set_size(rows, cols, slices); + } for (size_t i = 0; i < slices; i++) { Initialize(W.slice(i), rows, cols); diff --git a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp index 92bd9b7ee9..c3778c23a9 100644 --- a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp +++ b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp @@ -33,8 +33,6 @@ #include #include -using namespace mlpack::math; - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -66,11 +64,14 @@ class LecunNormalInitialization const size_t rows, const size_t cols) { - double_t variance = 1 / rows; + // He initialization rule says to initialize weights with random + // values taken from a gaussian distribution with mean = 0 and + // standard deviation = sqrt(1/rows), i.e. variance = (1/rows). + double_t variance = 1.0 / rows; if (W.is_empty()) { - W = arma::mat(rows, cols); + W.set_size(rows, cols); } W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } ); @@ -90,7 +91,10 @@ class LecunNormalInitialization const size_t cols, const size_t slices) { - W = arma::cube(rows, cols, slices); + if (W.is_empty()) + { + W.set_size(rows, cols, slices); + } for (size_t i = 0; i < slices; i++) { Initialize(W.slice(i), rows, cols); From b033c5acb49e06947364eb1673effff306bc9865 Mon Sep 17 00:00:00 2001 From: dakshitagrawal97 Date: Thu, 1 Mar 2018 07:32:18 +0530 Subject: [PATCH 08/15] fix type casting error --- src/mlpack/methods/ann/init_rules/he_init.hpp | 2 +- src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp index 8d569f725b..b12dec7d82 100644 --- a/src/mlpack/methods/ann/init_rules/he_init.hpp +++ b/src/mlpack/methods/ann/init_rules/he_init.hpp @@ -62,7 +62,7 @@ class HeInitialization // He initialization rule says to initialize weights with random // values taken from a gaussian distribution with mean = 0 and // standard deviation = sqrt(2/rows), i.e. variance = (2/rows). - double_t variance = 2.0 / rows; + double_t variance = 2.0 / ((double) rows); if (W.is_empty()) { diff --git a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp index c3778c23a9..fac030af9a 100644 --- a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp +++ b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp @@ -67,7 +67,7 @@ class LecunNormalInitialization // He initialization rule says to initialize weights with random // values taken from a gaussian distribution with mean = 0 and // standard deviation = sqrt(1/rows), i.e. variance = (1/rows). - double_t variance = 1.0 / rows; + double_t variance = 1.0 / ((double) rows); if (W.is_empty()) { From b5778d00a969c074ed76738571b11227d52868e8 Mon Sep 17 00:00:00 2001 From: dakshitagrawal97 Date: Sat, 3 Mar 2018 23:20:54 +0530 Subject: [PATCH 09/15] make changes in documentation and code --- src/mlpack/methods/ann/init_rules/he_init.hpp | 23 ++++++++++++++++--- .../ann/init_rules/lecun_normal_init.hpp | 23 +++++++++++++++++-- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp index b12dec7d82..7b96fcb379 100644 --- a/src/mlpack/methods/ann/init_rules/he_init.hpp +++ b/src/mlpack/methods/ann/init_rules/he_init.hpp @@ -33,7 +33,22 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * This class is used to initialize weight matrix with the He initialization rule. + * This class is used to initialize weight matrix with the He + * initialization rule given by He et. al. for neural networks. The He + * initialization initializes weights of the neural network to better + * suit the rectified activation units. + * + * For more information, the following paper can be referred to: + * + * @code + * @article{He2015DelvingDI, + * title={Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification}, + * author={Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun}, + * journal={2015 IEEE International Conference on Computer Vision (ICCV)}, + * year={2015}, + * pages={1026-1034}} + * @endcode + * */ class HeInitialization { @@ -62,14 +77,16 @@ class HeInitialization // He initialization rule says to initialize weights with random // values taken from a gaussian distribution with mean = 0 and // standard deviation = sqrt(2/rows), i.e. variance = (2/rows). - double_t variance = 2.0 / ((double) rows); + double variance = 2.0 / ((double) rows); if (W.is_empty()) { W.set_size(rows, cols); } - W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } ); + // Multipling a random variable X with variance V(X) by some factor c, + // then the variance V(cX) = (c^2)* V(X). + W.imbue( [&]() { return sqrt(variance) * arma::randn();} ); } /** diff --git a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp index fac030af9a..27c5205fbe 100644 --- a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp +++ b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp @@ -39,6 +39,23 @@ namespace ann /** Artificial Neural Network. */ { /** * This class is used to initialize weight matrix with the Lecun Normalization * initialization rule. + * + * For more information, the following papers can be referred to: + * + * @code + * @inproceedings{conf/nips/KlambauerUMH17, + * title = {Self-Normalizing Neural Networks.}, + * author = {Klambauer, Günter and Unterthiner, Thomas and Mayr, Andreas and Hochreiter, Sepp}, + * pages = {972-981}, + * year = 2017} + * + * @inproceedings{LeCun:1998:EB:645754.668382, + * title = {Efficient BackProp}, + * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B. and M\"{u}ller, Klaus-Robert}, + * year = {1998}, + * pages = {9--50}} + * @endcode + * */ class LecunNormalInitialization { @@ -67,14 +84,16 @@ class LecunNormalInitialization // He initialization rule says to initialize weights with random // values taken from a gaussian distribution with mean = 0 and // standard deviation = sqrt(1/rows), i.e. variance = (1/rows). - double_t variance = 1.0 / ((double) rows); + double variance = 1.0 / ((double) rows); if (W.is_empty()) { W.set_size(rows, cols); } - W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } ); + // Multipling a random variable X with variance V(X) by some factor c, + // then the variance V(cX) = (c^2)* V(X). + W.imbue( [&]() { return sqrt(variance) * arma::randn();} ); } /** From 97de396c680bb3ed9ae1eb99b4841476a6845d2c Mon Sep 17 00:00:00 2001 From: dakshitagrawal97 Date: Sat, 3 Mar 2018 23:29:43 +0530 Subject: [PATCH 10/15] fix indentation error --- src/mlpack/methods/ann/init_rules/he_init.hpp | 110 ++++++++-------- .../ann/init_rules/lecun_normal_init.hpp | 120 +++++++++--------- 2 files changed, 118 insertions(+), 112 deletions(-) diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp index 7b96fcb379..f0f38202f6 100644 --- a/src/mlpack/methods/ann/init_rules/he_init.hpp +++ b/src/mlpack/methods/ann/init_rules/he_init.hpp @@ -10,7 +10,8 @@ * * @code * @article{He2015DelvingDI, - * title={Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification}, + * title={Delving Deep into Rectifiers: Surpassing Human-Level Performance + * on ImageNet Classification}, * author={Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun}, * journal={2015 IEEE International Conference on Computer Vision (ICCV)}, * year={2015}, @@ -42,7 +43,8 @@ namespace ann /** Artificial Neural Network. */ { * * @code * @article{He2015DelvingDI, - * title={Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification}, + * title={Delving Deep into Rectifiers: Surpassing Human-Level Performance + * on ImageNet Classification}, * author={Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun}, * journal={2015 IEEE International Conference on Computer Vision (ICCV)}, * year={2015}, @@ -53,65 +55,65 @@ namespace ann /** Artificial Neural Network. */ { class HeInitialization { public: - /** - * Initialize the HeInitialization object. - * - */ - HeInitialization() + /** + * Initialize the HeInitialization object. + * + */ + HeInitialization() + { + // Nothing to do here. + } + + /** + * Initialize the elements of the weight matrix with the He initialization + * rule. + * + * @param W Weight matrix to initialize. + * @param rows Number of rows. + * @param cols Number of columns. + */ + void Initialize(arma::mat& W, + const size_t rows, + const size_t cols) + { + // He initialization rule says to initialize weights with random + // values taken from a gaussian distribution with mean = 0 and + // standard deviation = sqrt(2/rows), i.e. variance = (2/rows). + double variance = 2.0 / ((double) rows); + + if (W.is_empty()) { - // Nothing to do here. + W.set_size(rows, cols); } - /** - * Initialize the elements of the weight matrix with the He initialization - * rule. - * - * @param W Weight matrix to initialize. - * @param rows Number of rows. - * @param cols Number of columns. - */ - void Initialize(arma::mat& W, - const size_t rows, - const size_t cols) + // Multipling a random variable X with variance V(X) by some factor c, + // then the variance V(cX) = (c^2)* V(X). + W.imbue( [&]() { return sqrt(variance) * arma::randn();} ); + } + + /** + * Initialize the elements of the specified weight 3rd order tensor + * with He initialization rule. + * + * @param W Weight matrix to initialize. + * @param rows Number of rows. + * @param cols Number of columns. + * @param slice Numbers of slices. + */ + void Initialize(arma::cube & W, + const size_t rows, + const size_t cols, + const size_t slices) + { + if (W.is_empty()) { - // He initialization rule says to initialize weights with random - // values taken from a gaussian distribution with mean = 0 and - // standard deviation = sqrt(2/rows), i.e. variance = (2/rows). - double variance = 2.0 / ((double) rows); - - if (W.is_empty()) - { - W.set_size(rows, cols); - } - - // Multipling a random variable X with variance V(X) by some factor c, - // then the variance V(cX) = (c^2)* V(X). - W.imbue( [&]() { return sqrt(variance) * arma::randn();} ); + W.set_size(rows, cols, slices); } - /** - * Initialize the elements of the specified weight 3rd order tensor - * with He initialization rule. - * - * @param W Weight matrix to initialize. - * @param rows Number of rows. - * @param cols Number of columns. - * @param slice Numbers of slices. - */ - void Initialize(arma::cube & W, - const size_t rows, - const size_t cols, - const size_t slices) - { - if (W.is_empty()) - { - W.set_size(rows, cols, slices); - } - - for (size_t i = 0; i < slices; i++) { - Initialize(W.slice(i), rows, cols); - } + for (size_t i = 0; i < slices; i++) { + Initialize(W.slice(i), rows, cols); } + } }; // class HeInitialization } // namespace ann diff --git a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp index 27c5205fbe..60d3f97bd4 100644 --- a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp +++ b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp @@ -10,13 +10,15 @@ * @code * @inproceedings{conf/nips/KlambauerUMH17, * title = {Self-Normalizing Neural Networks.}, - * author = {Klambauer, Günter and Unterthiner, Thomas and Mayr, Andreas and Hochreiter, Sepp}, + * author = {Klambauer, Günter and Unterthiner, Thomas + * and Mayr, Andreas and Hochreiter, Sepp}, * pages = {972-981}, * year = 2017} * * @inproceedings{LeCun:1998:EB:645754.668382, * title = {Efficient BackProp}, - * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B. and M\"{u}ller, Klaus-Robert}, + * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B. + * and M\"{u}ller, Klaus-Robert}, * year = {1998}, * pages = {9--50}} * @endcode @@ -37,21 +39,23 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** -* This class is used to initialize weight matrix with the Lecun Normalization -* initialization rule. + * This class is used to initialize weight matrix with the Lecun Normalization + * initialization rule. * * For more information, the following papers can be referred to: * * @code * @inproceedings{conf/nips/KlambauerUMH17, * title = {Self-Normalizing Neural Networks.}, - * author = {Klambauer, Günter and Unterthiner, Thomas and Mayr, Andreas and Hochreiter, Sepp}, + * author = {Klambauer, Günter and Unterthiner, Thomas + * and Mayr, Andreas and Hochreiter, Sepp}, * pages = {972-981}, * year = 2017} * * @inproceedings{LeCun:1998:EB:645754.668382, * title = {Efficient BackProp}, - * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B. and M\"{u}ller, Klaus-Robert}, + * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B. + * and M\"{u}ller, Klaus-Robert}, * year = {1998}, * pages = {9--50}} * @endcode @@ -60,65 +64,65 @@ namespace ann /** Artificial Neural Network. */ { class LecunNormalInitialization { public: - /** - * Initialize the LecunNormalInitialization object. - * - */ - LecunNormalInitialization() + /** + * Initialize the LecunNormalInitialization object. + * + */ + LecunNormalInitialization() + { + // Nothing to do here. + } + + /** + * Initialize the elements of the weight matrix with the Lecun + * Normal initialization rule. + * + * @param W Weight matrix to initialize. + * @param rows Number of rows. + * @param cols Number of columns. + */ + void Initialize(arma::mat& W, + const size_t rows, + const size_t cols) + { + // He initialization rule says to initialize weights with random + // values taken from a gaussian distribution with mean = 0 and + // standard deviation = sqrt(1/rows), i.e. variance = (1/rows). + double variance = 1.0 / ((double) rows); + + if (W.is_empty()) { - // Nothing to do here. + W.set_size(rows, cols); } - /** - * Initialize the elements of the weight matrix with the Lecun - * Normal initialization rule. - * - * @param W Weight matrix to initialize. - * @param rows Number of rows. - * @param cols Number of columns. - */ - void Initialize(arma::mat& W, - const size_t rows, - const size_t cols) + // Multipling a random variable X with variance V(X) by some factor c, + // then the variance V(cX) = (c^2)* V(X). + W.imbue( [&]() { return sqrt(variance) * arma::randn();} ); + } + + /** + * Initialize the elements of the specified weight 3rd order tensor + * with Lecun Normal initialization rule. + * + * @param W Weight matrix to initialize. + * @param rows Number of rows. + * @param cols Number of columns. + * @param slice Numbers of slices. + */ + void Initialize(arma::cube & W, + const size_t rows, + const size_t cols, + const size_t slices) + { + if (W.is_empty()) { - // He initialization rule says to initialize weights with random - // values taken from a gaussian distribution with mean = 0 and - // standard deviation = sqrt(1/rows), i.e. variance = (1/rows). - double variance = 1.0 / ((double) rows); - - if (W.is_empty()) - { - W.set_size(rows, cols); - } - - // Multipling a random variable X with variance V(X) by some factor c, - // then the variance V(cX) = (c^2)* V(X). - W.imbue( [&]() { return sqrt(variance) * arma::randn();} ); + W.set_size(rows, cols, slices); } - /** - * Initialize the elements of the specified weight 3rd order tensor - * with Lecun Normal initialization rule. - * - * @param W Weight matrix to initialize. - * @param rows Number of rows. - * @param cols Number of columns. - * @param slice Numbers of slices. - */ - void Initialize(arma::cube & W, - const size_t rows, - const size_t cols, - const size_t slices) - { - if (W.is_empty()) - { - W.set_size(rows, cols, slices); - } - - for (size_t i = 0; i < slices; i++) { - Initialize(W.slice(i), rows, cols); - } + for (size_t i = 0; i < slices; i++) { + Initialize(W.slice(i), rows, cols); } + } }; // class LecunNormalInitialization } // namespace ann From d0839d12586ee7283c7d99c29125f2fae68a870f Mon Sep 17 00:00:00 2001 From: Prabhat Date: Fri, 30 Mar 2018 21:21:22 +0530 Subject: [PATCH 11/15] Minor style changes --- src/mlpack/methods/ann/init_rules/he_init.hpp | 23 +++------------ .../ann/init_rules/lecun_normal_init.hpp | 29 +++---------------- src/mlpack/tests/init_rules_test.cpp | 1 - 3 files changed, 8 insertions(+), 45 deletions(-) diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp index f0f38202f6..b0ee3c62b7 100644 --- a/src/mlpack/methods/ann/init_rules/he_init.hpp +++ b/src/mlpack/methods/ann/init_rules/he_init.hpp @@ -1,23 +1,11 @@ /** * @file he_init.hpp - * @author Dakshit Agrawal + * @authors Dakshit Agrawal and Prabhat Sharma * * Intialization rule given by He et. al. for neural networks. The He * initialization initializes weights of the neural network to better * suit the rectified activation units. * - * For more information, the following paper can be referred to: - * - * @code - * @article{He2015DelvingDI, - * title={Delving Deep into Rectifiers: Surpassing Human-Level Performance - * on ImageNet Classification}, - * author={Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun}, - * journal={2015 IEEE International Conference on Computer Vision (ICCV)}, - * year={2015}, - * pages={1026-1034}} - * @endcode - * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the * 3-clause BSD license along with mlpack. If not, see @@ -79,7 +67,7 @@ class HeInitialization // He initialization rule says to initialize weights with random // values taken from a gaussian distribution with mean = 0 and // standard deviation = sqrt(2/rows), i.e. variance = (2/rows). - double variance = 2.0 / ((double) rows); + const double variance = 2.0 / (double)rows; if (W.is_empty()) { @@ -88,7 +76,7 @@ class HeInitialization // Multipling a random variable X with variance V(X) by some factor c, // then the variance V(cX) = (c^2)* V(X). - W.imbue( [&]() { return sqrt(variance) * arma::randn();} ); + W.imbue( [&]() { return sqrt(variance) * arma::randn(); } ); } /** @@ -106,13 +94,10 @@ class HeInitialization const size_t slices) { if (W.is_empty()) - { W.set_size(rows, cols, slices); - } - for (size_t i = 0; i < slices; i++) { + for (size_t i = 0; i < slices; i++) Initialize(W.slice(i), rows, cols); - } } }; // class HeInitialization diff --git a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp index 60d3f97bd4..4a1bc9ce04 100644 --- a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp +++ b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp @@ -1,28 +1,10 @@ /** * @file lecun_normal_init.hpp - * @author Dakshit Agrawal + * @author Dakshit Agrawal and Prabhat Sharma * * Intialization rule given by Lecun et. al. for neural networks and * also mentioned in Self Normalizing Networks. * - * For more information, the following papers can be referred to: - * - * @code - * @inproceedings{conf/nips/KlambauerUMH17, - * title = {Self-Normalizing Neural Networks.}, - * author = {Klambauer, Günter and Unterthiner, Thomas - * and Mayr, Andreas and Hochreiter, Sepp}, - * pages = {972-981}, - * year = 2017} - * - * @inproceedings{LeCun:1998:EB:645754.668382, - * title = {Efficient BackProp}, - * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B. - * and M\"{u}ller, Klaus-Robert}, - * year = {1998}, - * pages = {9--50}} - * @endcode - * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the * 3-clause BSD license along with mlpack. If not, see @@ -88,7 +70,7 @@ class LecunNormalInitialization // He initialization rule says to initialize weights with random // values taken from a gaussian distribution with mean = 0 and // standard deviation = sqrt(1/rows), i.e. variance = (1/rows). - double variance = 1.0 / ((double) rows); + const double variance = 1.0 / ((double) rows); if (W.is_empty()) { @@ -97,7 +79,7 @@ class LecunNormalInitialization // Multipling a random variable X with variance V(X) by some factor c, // then the variance V(cX) = (c^2)* V(X). - W.imbue( [&]() { return sqrt(variance) * arma::randn();} ); + W.imbue( [&]() { return sqrt(variance) * arma::randn(); } ); } /** @@ -115,13 +97,10 @@ class LecunNormalInitialization const size_t slices) { if (W.is_empty()) - { W.set_size(rows, cols, slices); - } - for (size_t i = 0; i < slices; i++) { + for (size_t i = 0; i < slices; i++) Initialize(W.slice(i), rows, cols); - } } }; // class LecunNormalInitialization diff --git a/src/mlpack/tests/init_rules_test.cpp b/src/mlpack/tests/init_rules_test.cpp index 98dea98c3a..5ebc180eaf 100644 --- a/src/mlpack/tests/init_rules_test.cpp +++ b/src/mlpack/tests/init_rules_test.cpp @@ -249,7 +249,6 @@ BOOST_AUTO_TEST_CASE(LecunNormalInitTest) BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices); } - /** * Simple test of the NetworkInitialization class, we test it with every * implemented initialization rule and make sure the output is reasonable. From 2727a78454690354ec7deff8f93b326bb3c8e930 Mon Sep 17 00:00:00 2001 From: Prabhat Date: Sat, 31 Mar 2018 03:43:54 +0530 Subject: [PATCH 12/15] Order of tests changed --- src/mlpack/tests/init_rules_test.cpp | 99 ++++++++++++++-------------- 1 file changed, 49 insertions(+), 50 deletions(-) diff --git a/src/mlpack/tests/init_rules_test.cpp b/src/mlpack/tests/init_rules_test.cpp index 93d1d57205..5d2677b26a 100644 --- a/src/mlpack/tests/init_rules_test.cpp +++ b/src/mlpack/tests/init_rules_test.cpp @@ -200,56 +200,6 @@ BOOST_AUTO_TEST_CASE(GaussianInitTest) BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices); } -/** -* Simple test of the HeInitialization class. -*/ -BOOST_AUTO_TEST_CASE(HeInitTest) -{ - const size_t rows = 4; - const size_t cols = 4; - const size_t slices = 2; - - arma::mat weights; - arma::cube weights3d; - - HeInitialization initializer; - - initializer.Initialize(weights, rows, cols); - initializer.Initialize(weights3d, rows, cols, slices); - - BOOST_REQUIRE_EQUAL(weights.n_rows, rows); - BOOST_REQUIRE_EQUAL(weights.n_cols, cols); - - BOOST_REQUIRE_EQUAL(weights3d.n_rows, rows); - BOOST_REQUIRE_EQUAL(weights3d.n_cols, cols); - BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices); -} - -/** -* Simple test of the LecunNormalInitialization class. -*/ -BOOST_AUTO_TEST_CASE(LecunNormalInitTest) -{ - const size_t rows = 4; - const size_t cols = 4; - const size_t slices = 2; - - arma::mat weights; - arma::cube weights3d; - - LecunNormalInitialization initializer; - - initializer.Initialize(weights, rows, cols); - initializer.Initialize(weights3d, rows, cols, slices); - - BOOST_REQUIRE_EQUAL(weights.n_rows, rows); - BOOST_REQUIRE_EQUAL(weights.n_cols, cols); - - BOOST_REQUIRE_EQUAL(weights3d.n_rows, rows); - BOOST_REQUIRE_EQUAL(weights3d.n_cols, cols); - BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices); -} - /** * Simple test of the NetworkInitialization class, we test it with every * implemented initialization rule and make sure the output is reasonable. @@ -380,5 +330,54 @@ BOOST_AUTO_TEST_CASE(GlorotInitNormalTest) BOOST_REQUIRE_EQUAL(weights3d.n_slices, 2); } +/** +* Simple test of the HeInitialization class. +*/ +BOOST_AUTO_TEST_CASE(HeInitTest) +{ + const size_t rows = 4; + const size_t cols = 4; + const size_t slices = 2; + + arma::mat weights; + arma::cube weights3d; + + HeInitialization initializer; + + initializer.Initialize(weights, rows, cols); + initializer.Initialize(weights3d, rows, cols, slices); + + BOOST_REQUIRE_EQUAL(weights.n_rows, rows); + BOOST_REQUIRE_EQUAL(weights.n_cols, cols); + + BOOST_REQUIRE_EQUAL(weights3d.n_rows, rows); + BOOST_REQUIRE_EQUAL(weights3d.n_cols, cols); + BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices); +} + +/** +* Simple test of the LecunNormalInitialization class. +*/ +BOOST_AUTO_TEST_CASE(LecunNormalInitTest) +{ + const size_t rows = 4; + const size_t cols = 4; + const size_t slices = 2; + + arma::mat weights; + arma::cube weights3d; + + LecunNormalInitialization initializer; + + initializer.Initialize(weights, rows, cols); + initializer.Initialize(weights3d, rows, cols, slices); + + BOOST_REQUIRE_EQUAL(weights.n_rows, rows); + BOOST_REQUIRE_EQUAL(weights.n_cols, cols); + + BOOST_REQUIRE_EQUAL(weights3d.n_rows, rows); + BOOST_REQUIRE_EQUAL(weights3d.n_cols, cols); + BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices); +} BOOST_AUTO_TEST_SUITE_END(); From c0c4e78588a5c70ffe302c1a2aed80529e4b71f3 Mon Sep 17 00:00:00 2001 From: Prabhat Date: Fri, 6 Apr 2018 03:14:09 +0530 Subject: [PATCH 13/15] Fix citation style --- .../ann/init_rules/lecun_normal_init.hpp | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp index 4a1bc9ce04..41ad1e9121 100644 --- a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp +++ b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp @@ -1,6 +1,7 @@ /** * @file lecun_normal_init.hpp - * @author Dakshit Agrawal and Prabhat Sharma + * @author Dakshit Agrawal + * @author Prabhat Sharma * * Intialization rule given by Lecun et. al. for neural networks and * also mentioned in Self Normalizing Networks. @@ -28,18 +29,18 @@ namespace ann /** Artificial Neural Network. */ { * * @code * @inproceedings{conf/nips/KlambauerUMH17, - * title = {Self-Normalizing Neural Networks.}, + * title = {Self-Normalizing Neural Networks.}, * author = {Klambauer, Günter and Unterthiner, Thomas - * and Mayr, Andreas and Hochreiter, Sepp}, - * pages = {972-981}, - * year = 2017} + * and Mayr, Andreas and Hochreiter, Sepp}, + * pages = {972-981}, + * year = 2017} * * @inproceedings{LeCun:1998:EB:645754.668382, - * title = {Efficient BackProp}, + * title = {Efficient BackProp}, * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B. - * and M\"{u}ller, Klaus-Robert}, - * year = {1998}, - * pages = {9--50}} + * and M\"{u}ller, Klaus-Robert}, + * year = {1998}, + * pages = {9--50}} * @endcode * */ @@ -69,7 +70,7 @@ class LecunNormalInitialization { // He initialization rule says to initialize weights with random // values taken from a gaussian distribution with mean = 0 and - // standard deviation = sqrt(1/rows), i.e. variance = (1/rows). + // standard deviation = sqrt(1 / rows), i.e. variance = (1 / rows). const double variance = 1.0 / ((double) rows); if (W.is_empty()) @@ -78,7 +79,7 @@ class LecunNormalInitialization } // Multipling a random variable X with variance V(X) by some factor c, - // then the variance V(cX) = (c^2)* V(X). + // then the variance V(cX) = (c ^ 2) * V(X). W.imbue( [&]() { return sqrt(variance) * arma::randn(); } ); } From 96480b97a85ec4c9491580f943bde67c9128c0c8 Mon Sep 17 00:00:00 2001 From: Prabhat Date: Sun, 8 Apr 2018 20:57:55 +0530 Subject: [PATCH 14/15] Fixed minor style issue Doxygen authors changed to author --- src/mlpack/methods/ann/init_rules/he_init.hpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp index b0ee3c62b7..d555464402 100644 --- a/src/mlpack/methods/ann/init_rules/he_init.hpp +++ b/src/mlpack/methods/ann/init_rules/he_init.hpp @@ -1,6 +1,7 @@ /** * @file he_init.hpp - * @authors Dakshit Agrawal and Prabhat Sharma + * @author Dakshit Agrawal + * @author Prabhat Sharma * * Intialization rule given by He et. al. for neural networks. The He * initialization initializes weights of the neural network to better @@ -31,12 +32,12 @@ namespace ann /** Artificial Neural Network. */ { * * @code * @article{He2015DelvingDI, - * title={Delving Deep into Rectifiers: Surpassing Human-Level Performance - * on ImageNet Classification}, - * author={Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun}, - * journal={2015 IEEE International Conference on Computer Vision (ICCV)}, - * year={2015}, - * pages={1026-1034}} + * title = {Delving Deep into Rectifiers: Surpassing Human-Level Performance + * on ImageNet Classification}, + * author = {Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun}, + * journal = {2015 IEEE International Conference on Computer Vision (ICCV)}, + * year = {2015}, + * pages = {1026-1034}} * @endcode * */ From cb1d2d4b88fa00544637cc193f8c4d29b27defbe Mon Sep 17 00:00:00 2001 From: Prabhat Date: Wed, 11 Apr 2018 02:48:11 +0530 Subject: [PATCH 15/15] Fix citation style --- src/mlpack/methods/ann/init_rules/he_init.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp index d555464402..822fb5a81e 100644 --- a/src/mlpack/methods/ann/init_rules/he_init.hpp +++ b/src/mlpack/methods/ann/init_rules/he_init.hpp @@ -32,12 +32,12 @@ namespace ann /** Artificial Neural Network. */ { * * @code * @article{He2015DelvingDI, - * title = {Delving Deep into Rectifiers: Surpassing Human-Level Performance - * on ImageNet Classification}, - * author = {Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun}, - * journal = {2015 IEEE International Conference on Computer Vision (ICCV)}, - * year = {2015}, - * pages = {1026-1034}} + * title = {Delving Deep into Rectifiers: Surpassing Human-Level Performance + * on ImageNet Classification}, + * author = {Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun}, + * journal = {2015 IEEE International Conference on Computer Vision (ICCV)}, + * year = {2015}, + * pages = {1026-1034}} * @endcode * */