Merge pull request #1342 from Prabhat-IIT/newinit

He Initialization and Lecun Normal initialization.

src/mlpack/methods/ann/init_rules/CMakeLists.txt

@@ -3,8 +3,10 @@
 set(SOURCES
   const_init.hpp
   gaussian_init.hpp
+  he_init.hpp
   init_rules_traits.hpp
   kathirvalavakumar_subavathi_init.hpp
+  lecun_normal_init.hpp
   network_init.hpp
   nguyen_widrow_init.hpp
   oivs_init.hpp

src/mlpack/methods/ann/init_rules/he_init.hpp

@@ -0,0 +1,108 @@
+/**
+ * @file he_init.hpp
+ * @author Dakshit Agrawal
+ * @author Prabhat Sharma
+ *
+ * Intialization rule given by He et. al. for neural networks. The He
+ * initialization initializes weights of the neural network to better
+ * suit the rectified activation units.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef MLPACK_METHODS_ANN_INIT_RULES_HE_INIT_HPP
+#define MLPACK_METHODS_ANN_INIT_RULES_HE_INIT_HPP
+
+#include <mlpack/prereqs.hpp>
+#include <mlpack/core/math/random.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * This class is used to initialize weight matrix with the He
+ * initialization rule given by He et. al. for neural networks. The He
+ * initialization initializes weights of the neural network to better
+ * suit the rectified activation units.
+ *
+ * For more information, the following paper can be referred to:
+ *
+ * @code
+ * @article{He2015DelvingDI,
+ *   title   = {Delving Deep into Rectifiers: Surpassing Human-Level Performance
+ *              on ImageNet Classification},
+ *   author  = {Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun},
+ *   journal = {2015 IEEE International Conference on Computer Vision (ICCV)},
+ *   year    = {2015},
+ *   pages   = {1026-1034}}
+ * @endcode
+ *
+ */
+class HeInitialization
+{
+ public:
+  /**
+   * Initialize the HeInitialization object.
+   *
+   */
+  HeInitialization()
+  {
+    // Nothing to do here.
+  }
+
+  /**
+   * Initialize the elements of the weight matrix with the He initialization
+   * rule.
+   *
+   * @param W Weight matrix to initialize.
+   * @param rows Number of rows.
+   * @param cols Number of columns.
+   */
+  void Initialize(arma::mat& W,
+                  const size_t rows,
+                  const size_t cols)
+  {
+    // He initialization rule says to initialize weights with random
+    // values taken from a gaussian distribution with mean = 0 and
+    // standard deviation = sqrt(2/rows), i.e. variance = (2/rows).
+    const double variance = 2.0 / (double)rows;
+
+    if (W.is_empty())
+    {
+      W.set_size(rows, cols);
+    }
+
+    // Multipling a random variable X with variance V(X) by some factor c,
+    // then the variance V(cX) = (c^2)* V(X).
+    W.imbue( [&]() { return sqrt(variance) * arma::randn(); } );
+  }
+
+  /**
+   * Initialize the elements of the specified weight 3rd order tensor
+   * with He initialization rule.
+   *
+   * @param W Weight matrix to initialize.
+   * @param rows Number of rows.
+   * @param cols Number of columns.
+   * @param slice Numbers of slices.
+   */
+  void Initialize(arma::cube & W,
+                  const size_t rows,
+                  const size_t cols,
+                  const size_t slices)
+  {
+    if (W.is_empty())
+      W.set_size(rows, cols, slices);
+
+    for (size_t i = 0; i < slices; i++)
+      Initialize(W.slice(i), rows, cols);
+  }
+}; // class HeInitialization
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp

@@ -0,0 +1,111 @@
+/**
+ * @file lecun_normal_init.hpp
+ * @author Dakshit Agrawal
+ * @author Prabhat Sharma
+ *
+ * Intialization rule given by Lecun et. al. for neural networks and
+ * also mentioned in Self Normalizing Networks.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef MLPACK_METHODS_ANN_INIT_RULES_LECUN_NORMAL_INIT_HPP
+#define MLPACK_METHODS_ANN_INIT_RULES_LECUN_NORMAL_INIT_HPP
+
+#include <mlpack/prereqs.hpp>
+#include <mlpack/core/math/random.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * This class is used to initialize weight matrix with the Lecun Normalization
+ * initialization rule.
+ *
+ * For more information, the following papers can be referred to:
+ *
+ * @code
+ * @inproceedings{conf/nips/KlambauerUMH17,
+ * title  = {Self-Normalizing Neural Networks.},
+ * author = {Klambauer, Günter and Unterthiner, Thomas
+ *           and Mayr, Andreas and Hochreiter, Sepp},
+ * pages  = {972-981},
+ * year   = 2017}
+ *
+ * @inproceedings{LeCun:1998:EB:645754.668382,
+ * title  = {Efficient BackProp},
+ * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B.
+ *          and M\"{u}ller, Klaus-Robert},
+ * year   = {1998},
+ * pages  = {9--50}}
+ * @endcode
+ *
+*/
+class LecunNormalInitialization
+{
+ public:
+  /**
+   * Initialize the LecunNormalInitialization object.
+   *
+   */
+  LecunNormalInitialization()
+  {
+    // Nothing to do here.
+  }
+
+  /**
+   * Initialize the elements of the weight matrix with the Lecun
+   * Normal initialization rule.
+   *
+   * @param W Weight matrix to initialize.
+   * @param rows Number of rows.
+   * @param cols Number of columns.
+   */
+  void Initialize(arma::mat& W,
+                  const size_t rows,
+                  const size_t cols)
+  {
+    // He initialization rule says to initialize weights with random
+    // values taken from a gaussian distribution with mean = 0 and
+    // standard deviation = sqrt(1 / rows), i.e. variance = (1 / rows).
+    const double variance = 1.0 / ((double) rows);
+
+    if (W.is_empty())
+    {
+      W.set_size(rows, cols);
+    }
+
+    // Multipling a random variable X with variance V(X) by some factor c,
+    // then the variance V(cX) = (c ^ 2) * V(X).
+    W.imbue( [&]() { return sqrt(variance) * arma::randn(); } );
+  }
+
+  /**
+   * Initialize the elements of the specified weight 3rd order tensor
+   * with Lecun Normal initialization rule.
+   *
+   * @param W Weight matrix to initialize.
+   * @param rows Number of rows.
+   * @param cols Number of columns.
+   * @param slice Numbers of slices.
+   */
+  void Initialize(arma::cube & W,
+                  const size_t rows,
+                  const size_t cols,
+                  const size_t slices)
+  {
+    if (W.is_empty())
+      W.set_size(rows, cols, slices);
+
+    for (size_t i = 0; i < slices; i++)
+      Initialize(W.slice(i), rows, cols);
+  }
+}; // class LecunNormalInitialization
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/tests/init_rules_test.cpp

@@ -24,6 +24,8 @@
 #include <mlpack/methods/ann/init_rules/const_init.hpp>
 #include <mlpack/methods/ann/init_rules/gaussian_init.hpp>
 #include <mlpack/methods/ann/init_rules/glorot_init.hpp>
+#include <mlpack/methods/ann/init_rules/he_init.hpp>
+#include <mlpack/methods/ann/init_rules/lecun_normal_init.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
@@ -328,5 +330,54 @@ BOOST_AUTO_TEST_CASE(GlorotInitNormalTest)
   BOOST_REQUIRE_EQUAL(weights3d.n_slices, 2);
 }
 
+/**
+* Simple test of the HeInitialization class.
+*/
+BOOST_AUTO_TEST_CASE(HeInitTest)
+{
+  const size_t rows = 4;
+  const size_t cols = 4;
+  const size_t slices = 2;
+
+  arma::mat weights;
+  arma::cube weights3d;
+
+  HeInitialization initializer;
+
+  initializer.Initialize(weights, rows, cols);
+  initializer.Initialize(weights3d, rows, cols, slices);
+
+  BOOST_REQUIRE_EQUAL(weights.n_rows, rows);
+  BOOST_REQUIRE_EQUAL(weights.n_cols, cols);
+
+  BOOST_REQUIRE_EQUAL(weights3d.n_rows, rows);
+  BOOST_REQUIRE_EQUAL(weights3d.n_cols, cols);
+  BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices);
+}
+
+/**
+* Simple test of the LecunNormalInitialization class.
+*/
+BOOST_AUTO_TEST_CASE(LecunNormalInitTest)
+{
+  const size_t rows = 4;
+  const size_t cols = 4;
+  const size_t slices = 2;
+
+  arma::mat weights;
+  arma::cube weights3d;
+
+  LecunNormalInitialization initializer;
+
+  initializer.Initialize(weights, rows, cols);
+  initializer.Initialize(weights3d, rows, cols, slices);
+
+  BOOST_REQUIRE_EQUAL(weights.n_rows, rows);
+  BOOST_REQUIRE_EQUAL(weights.n_cols, cols);
+
+  BOOST_REQUIRE_EQUAL(weights3d.n_rows, rows);
+  BOOST_REQUIRE_EQUAL(weights3d.n_cols, cols);
+  BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices);
+}
 
 BOOST_AUTO_TEST_SUITE_END();