From b2c4bcfb174e4d4ee2bcf21f5be5784e52e05f1c Mon Sep 17 00:00:00 2001
From: dakshitagrawal97 <neelagrawal08@gmail.com>
Date: Tue, 27 Feb 2018 01:19:05 +0530
Subject: [PATCH 01/67] Add He initialization rule

---
 src/mlpack/methods/ann/init_rules/he_init.hpp | 100 ++++++++++++++++++
 1 file changed, 100 insertions(+)
 create mode 100644 src/mlpack/methods/ann/init_rules/he_init.hpp

diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp
new file mode 100644
index 0000000000..58e9732b3b
--- /dev/null
+++ b/src/mlpack/methods/ann/init_rules/he_init.hpp
@@ -0,0 +1,100 @@
+/**
+ * @file he_init.hpp
+ * @author Dakshit Agrawal
+ *
+ * Intialization rule given by He et. al. for neural networks. The He
+ * initialization initializes weights of the neural network to better
+ * suit the rectified activation units.
+ *
+ * For more information, the following paper can be referred to:
+ *
+ * @code
+ * @article{He2015DelvingDI,
+ * title={Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification},
+ * author={Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun},
+ * journal={2015 IEEE International Conference on Computer Vision (ICCV)},
+ * year={2015},
+ * pages={1026-1034}}
+ * @endcode
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef MLPACK_METHODS_ANN_INIT_RULES_HE_INIT_HPP
+#define MLPACK_METHODS_ANN_INIT_RULES_HE_INIT_HPP
+
+#include <mlpack/prereqs.hpp>
+#include <mlpack/core/math/random.hpp>
+
+using namespace mlpack::math;
+
+namespace mlpack {
+    namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * This class is used to initialize weight matrix with the He initialization rule.
+ */
+        class HeInitialization
+        {
+        public:
+            /**
+             * Initialize the HeInitialization object.
+             *
+             */
+            HeInitialization()
+            {
+                // Nothing to do here.
+            }
+
+            /**
+             * Initialize the elements of the weight matrix with the He initialization
+             * rule.
+             *
+             * @param W Weight matrix to initialize.
+             * @param rows Number of rows.
+             * @param cols Number of columns.
+             */
+            void Initialize(arma::mat& W,
+                            const size_t rows,
+                            const size_t cols)
+            {
+                double_t variance = 2 / rows;
+
+                if (W.is_empty())
+                {
+                    W = arma::mat(rows, cols);
+                }
+
+                W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } );
+            }
+
+            /**
+             * Initialize the elements of the specified weight 3rd order tensor
+             * with He initialization rule.
+             *
+             * @param W Weight matrix to initialize.
+             * @param rows Number of rows.
+             * @param cols Number of columns.
+             * @param slice Numbers of slices.
+             */
+            void Initialize(arma::cube & W,
+                            const size_t rows,
+                            const size_t cols,
+                            const size_t slices)
+            {
+                W = arma::cube(rows, cols, slices);
+
+                for (size_t i = 0; i < slices; i++) {
+                    Initialize(W.slice(i), rows, cols);
+                }
+            }
+
+        }; // class HeInitialization
+
+    } // namespace ann
+} // namespace mlpack
+
+#endif
\ No newline at end of file

From 0f2ff46f7344e309701895123e14469e0bdb514f Mon Sep 17 00:00:00 2001
From: dakshitagrawal97 <neelagrawal08@gmail.com>
Date: Tue, 27 Feb 2018 01:20:20 +0530
Subject: [PATCH 02/67] Add Lecun Normal initialization rule.

---
 .../ann/init_rules/lecun_normal_init.hpp      | 105 ++++++++++++++++++
 1 file changed, 105 insertions(+)
 create mode 100644 src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp

diff --git a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
new file mode 100644
index 0000000000..14cf63d07d
--- /dev/null
+++ b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
@@ -0,0 +1,105 @@
+/**
+ * @file lecun_normal_init.hpp
+ * @author Dakshit Agrawal
+ *
+ * Intialization rule given by Lecun et. al. for neural networks and
+ * also mentioned in Self Normalizing Networks.
+ *
+ * For more information, the following papers can be referred to:
+ *
+ * @code
+ * @inproceedings{conf/nips/KlambauerUMH17,
+ * title = {Self-Normalizing Neural Networks.},
+ * author = {Klambauer, Günter and Unterthiner, Thomas and Mayr, Andreas and Hochreiter, Sepp},
+ * pages = {972-981},
+ * year = 2017}
+ *
+ * @inproceedings{LeCun:1998:EB:645754.668382,
+ * title = {Efficient BackProp},
+ * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B. and M\"{u}ller, Klaus-Robert},
+ * year = {1998},
+ * pages = {9--50}}
+ * @endcode
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef MLPACK_METHODS_ANN_INIT_RULES_LECUN_NORMAL_INIT_HPP
+#define MLPACK_METHODS_ANN_INIT_RULES_LECUN_NORMAL_INIT_HPP
+
+#include <mlpack/prereqs.hpp>
+#include <mlpack/core/math/random.hpp>
+
+using namespace mlpack::math;
+
+namespace mlpack {
+    namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * This class is used to initialize weight matrix with the Lecun Normalization
+ * initialization rule.
+ */
+        class LecunNormalInitialization
+        {
+        public:
+            /**
+             * Initialize the LecunNormalInitialization object.
+             *
+             */
+            LecunNormalInitialization()
+            {
+                // Nothing to do here.
+            }
+
+            /**
+             * Initialize the elements of the weight matrix with the Lecun
+             * Normal initialization rule.
+             *
+             * @param W Weight matrix to initialize.
+             * @param rows Number of rows.
+             * @param cols Number of columns.
+             */
+            void Initialize(arma::mat& W,
+                            const size_t rows,
+                            const size_t cols)
+            {
+                double_t variance = 1 / rows;
+
+                if (W.is_empty())
+                {
+                    W = arma::mat(rows, cols);
+                }
+
+                W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } );
+            }
+
+            /**
+             * Initialize the elements of the specified weight 3rd order tensor
+             * with Lecun Normal initialization rule.
+             *
+             * @param W Weight matrix to initialize.
+             * @param rows Number of rows.
+             * @param cols Number of columns.
+             * @param slice Numbers of slices.
+             */
+            void Initialize(arma::cube & W,
+                            const size_t rows,
+                            const size_t cols,
+                            const size_t slices)
+            {
+                W = arma::cube(rows, cols, slices);
+
+                for (size_t i = 0; i < slices; i++) {
+                    Initialize(W.slice(i), rows, cols);
+                }
+            }
+
+        }; // class LecunNormalInitialization
+
+    } // namespace ann
+} // namespace mlpack
+
+#endif
\ No newline at end of file

From 8f471a86cd3720ca90e6d7e5a780886b1c2578ae Mon Sep 17 00:00:00 2001
From: dakshitagrawal97 <neelagrawal08@gmail.com>
Date: Tue, 27 Feb 2018 01:22:53 +0530
Subject: [PATCH 03/67] update CMakeLists

---
 src/mlpack/methods/ann/init_rules/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/mlpack/methods/ann/init_rules/CMakeLists.txt b/src/mlpack/methods/ann/init_rules/CMakeLists.txt
index e8172df613..83531f9191 100644
--- a/src/mlpack/methods/ann/init_rules/CMakeLists.txt
+++ b/src/mlpack/methods/ann/init_rules/CMakeLists.txt
@@ -3,8 +3,10 @@
 set(SOURCES
   const_init.hpp
   gaussian_init.hpp
+  he_init.hpp
   init_rules_traits.hpp
   kathirvalavakumar_subavathi_init.hpp
+  lecun_normal_init.hpp
   network_init.hpp
   nguyen_widrow_init.hpp
   oivs_init.hpp

From dd8ac49375b005e707d2142a4423d0f61f9a2dda Mon Sep 17 00:00:00 2001
From: dakshitagrawal97 <neelagrawal08@gmail.com>
Date: Tue, 27 Feb 2018 07:17:03 +0530
Subject: [PATCH 04/67] remove unnecessary rescale boolean variable from
 dropout layer

---
 src/mlpack/methods/ann/layer/dropout.hpp      | 176 +++++++++---------
 src/mlpack/methods/ann/layer/dropout_impl.hpp | 104 +++++------
 2 files changed, 128 insertions(+), 152 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/dropout.hpp b/src/mlpack/methods/ann/layer/dropout.hpp
index d562b85cf1..55e084bc3b 100644
--- a/src/mlpack/methods/ann/layer/dropout.hpp
+++ b/src/mlpack/methods/ann/layer/dropout.hpp
@@ -3,7 +3,7 @@
  * @author Marcus Edel
  *
  * Definition of the Dropout class, which implements a regularizer that
- * randomly sets units to zero. Preventing units from co-adapting.
+ * randomly sets units to zero preventing units from co-adapting.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -16,14 +16,13 @@
 #include <mlpack/prereqs.hpp>
 
 namespace mlpack {
-namespace ann /** Artificial Neural Network. */ {
+    namespace ann /** Artificial Neural Network. */ {
 
 /**
- * The dropout layer is a regularizer that randomly with probability ratio
+ * The dropout layer is a regularizer that randomly with probability 'ratio'
  * sets input values to zero and scales the remaining elements by factor 1 /
- * (1 - ratio). If rescale is true the input is scaled with 1 / (1-p) when
- * deterministic is false. In the deterministic mode (during testing), the layer
- * just scales the output.
+ * (1 - ratio) rather than during test time so as to keep the expected sum same.
+ * In the deterministic mode (during testing), there is no change in the input.
  *
  * Note: During training you should set deterministic to false and during
  * testing you should set deterministic to true.
@@ -47,111 +46,104 @@ namespace ann /** Artificial Neural Network. */ {
  * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
  */
-template <
-    typename InputDataType = arma::mat,
-    typename OutputDataType = arma::mat
->
-class Dropout
-{
- public:
-  /**
-   * Create the Dropout object using the specified ratio and rescale
-   * parameter.
-   *
-   * @param ratio The probability of setting a value to zero.
-   * @param rescale If true the input is rescaled when deterministic is False.
-   */
-  Dropout(const double ratio = 0.5, const bool rescale = true);
+        template<
+                typename InputDataType = arma::mat,
+                typename OutputDataType = arma::mat
+        >
+        class Dropout {
+        public:
+            /**
+             * Create the Dropout object using the specified ratio parameter.
+             *
+             * @param ratio The probability of setting a value to zero.
+             */
+            Dropout(const double ratio = 0.5);
 
-  /**
-   * Ordinary feed forward pass of the dropout layer.
-   *
-   * @param input Input data used for evaluating the specified function.
-   * @param output Resulting output activation.
-   */
-  template<typename eT>
-  void Forward(const arma::Mat<eT>&& input, arma::Mat<eT>&& output);
+            /**
+             * Ordinary feed forward pass of the dropout layer.
+             *
+             * @param input Input data used for evaluating the specified function.
+             * @param output Resulting output activation.
+             */
+            template<typename eT>
+            void Forward(const arma::Mat<eT> &&input, arma::Mat<eT> &&output);
 
-  /**
-   * Ordinary feed backward pass of the dropout layer.
-   *
-   * @param input The propagated input activation.
-   * @param gy The backpropagated error.
-   * @param g The calculated gradient.
-   */
-  template<typename eT>
-  void Backward(const arma::Mat<eT>&& /* input */,
-                arma::Mat<eT>&& gy,
-                arma::Mat<eT>&& g);
+            /**
+             * Ordinary feed backward pass of the dropout layer.
+             *
+             * @param input The propagated input activation.
+             * @param gy The backpropagated error.
+             * @param g The calculated gradient.
+             */
+            template<typename eT>
+            void Backward(const arma::Mat<eT> && /* input */,
+                          arma::Mat<eT> &&gy,
+                          arma::Mat<eT> &&g);
 
-  //! Get the input parameter.
-  InputDataType const& InputParameter() const { return inputParameter; }
-  //! Modify the input parameter.
-  InputDataType& InputParameter() { return inputParameter; }
+            //! Get the input parameter.
+            InputDataType const &InputParameter() const { return inputParameter; }
 
-  //! Get the output parameter.
-  OutputDataType const& OutputParameter() const { return outputParameter; }
-  //! Modify the output parameter.
-  OutputDataType& OutputParameter() { return outputParameter; }
+            //! Modify the input parameter.
+            InputDataType &InputParameter() { return inputParameter; }
 
-  //! Get the detla.
-  OutputDataType const& Delta() const { return delta; }
-  //! Modify the delta.
-  OutputDataType& Delta() { return delta; }
+            //! Get the output parameter.
+            OutputDataType const &OutputParameter() const { return outputParameter; }
 
-  //! The value of the deterministic parameter.
-  bool Deterministic() const { return deterministic; }
-  //! Modify the value of the deterministic parameter.
-  bool& Deterministic() { return deterministic; }
+            //! Modify the output parameter.
+            OutputDataType &OutputParameter() { return outputParameter; }
 
-  //! The probability of setting a value to zero.
-  double Ratio() const { return ratio; }
+            //! Get the detla.
+            OutputDataType const &Delta() const { return delta; }
 
-  //! Modify the probability of setting a value to zero.
-  void Ratio(const double r)
-  {
-    ratio = r;
-    scale = 1.0 / (1.0 - ratio);
-  }
+            //! Modify the delta.
+            OutputDataType &Delta() { return delta; }
 
-  //! The value of the rescale parameter.
-  bool Rescale() const {return rescale; }
-  //! Modify the value of the rescale parameter.
-  bool& Rescale() {return rescale; }
+            //! The value of the deterministic parameter.
+            bool Deterministic() const { return deterministic; }
 
-  /**
-   * Serialize the layer.
-   */
-  template<typename Archive>
-  void serialize(Archive& ar, const unsigned int /* version */);
+            //! Modify the value of the deterministic parameter.
+            bool &Deterministic() { return deterministic; }
 
- private:
-  //! Locally-stored delta object.
-  OutputDataType delta;
+            //! The probability of setting a value to zero.
+            double Ratio() const { return ratio; }
 
-  //! Locally-stored input parameter object.
-  InputDataType inputParameter;
+            //! Modify the probability of setting a value to zero.
+            void Ratio(const double r) {
+                ratio = r;
+                scale = 1.0 / (1.0 - ratio);
+            }
 
-  //! Locally-stored output parameter object.
-  OutputDataType outputParameter;
+            /**
+             * Serialize the layer.
+             */
+            template<typename Archive>
+            void serialize(Archive &ar, const unsigned int /* version */);
 
-  //! Locally-stored mast object.
-  OutputDataType mask;
+        private:
+            //! Locally-stored delta object.
+            OutputDataType delta;
 
-  //! The probability of setting a value to zero.
-  double ratio;
+            //! Locally-stored input parameter object.
+            InputDataType inputParameter;
 
-  //! The scale fraction.
-  double scale;
+            //! Locally-stored output parameter object.
+            OutputDataType outputParameter;
 
-  //! If true dropout and scaling is disabled, see notes above.
-  bool deterministic;
+            //! Locally-stored mast object.
+            OutputDataType mask;
 
-  //! If true the input is rescaled when deterministic is False.
-  bool rescale;
-}; // class Dropout
+            //! The probability of setting a value to zero.
+            double ratio;
 
-} // namespace ann
+            //! The scale fraction.
+            double scale;
+
+            //! If true dropout and scaling is disabled, see notes above.
+            bool deterministic;
+
+        }; // class Dropout
+
+    } // namespace ann
 } // namespace mlpack
 
 // Include implementation.
diff --git a/src/mlpack/methods/ann/layer/dropout_impl.hpp b/src/mlpack/methods/ann/layer/dropout_impl.hpp
index 2f7c03c6e9..9540fc2cd7 100644
--- a/src/mlpack/methods/ann/layer/dropout_impl.hpp
+++ b/src/mlpack/methods/ann/layer/dropout_impl.hpp
@@ -17,72 +17,56 @@
 #include "dropout.hpp"
 
 namespace mlpack {
-namespace ann /** Artificial Neural Network. */ {
+    namespace ann /** Artificial Neural Network. */ {
 
-template<typename InputDataType, typename OutputDataType>
-Dropout<InputDataType, OutputDataType>::Dropout(
-    const double ratio, const bool rescale) :
-    ratio(ratio),
-    scale(1.0 / (1.0 - ratio)),
-    deterministic(true),
-    rescale(rescale)
-{
-  // Nothing to do here.
-}
+        template<typename InputDataType, typename OutputDataType>
+        Dropout<InputDataType, OutputDataType>::Dropout(
+                const double ratio) :
+                ratio(ratio),
+                scale(1.0 / (1.0 - ratio)),
+                deterministic(false) {
+            // Nothing to do here.
+        }
 
-template<typename InputDataType, typename OutputDataType>
-template<typename eT>
-void Dropout<InputDataType, OutputDataType>::Forward(
-    const arma::Mat<eT>&& input,
-    arma::Mat<eT>&& output)
-{
-  // The dropout mask will not be multiplied in the deterministic mode
-  // (during testing).
-  if (deterministic)
-  {
-    if (!rescale)
-    {
-      output = input;
-    }
-    else
-    {
-      output = input * scale;
-    }
-  }
-  else
-  {
-    // Scale with input / (1 - ratio) and set values to zero with probability
-    // ratio.
-    mask = arma::randu<arma::Mat<eT> >(input.n_rows, input.n_cols);
-    mask.transform( [&](double val) { return (val > ratio); } );
-    output = input % mask * scale;
-  }
-}
+        template<typename InputDataType, typename OutputDataType>
+        template<typename eT>
+        void Dropout<InputDataType, OutputDataType>::Forward(
+                const arma::Mat<eT> &&input,
+                arma::Mat<eT> &&output) {
+            // The dropout mask will not be multiplied in the deterministic mode
+            // (during testing).
+            if (deterministic) {
+                output = input;
+            } else {
+                // Scale with input / (1 - ratio) and set values to zero with probability
+                // ratio.
+                mask = arma::randu < arma::Mat<eT> > (input.n_rows, input.n_cols);
+                mask.transform([&](double val) { return (val > ratio); });
+                output = input % mask * scale;
+            }
+        }
 
-template<typename InputDataType, typename OutputDataType>
-template<typename eT>
-void Dropout<InputDataType, OutputDataType>::Backward(
-    const arma::Mat<eT>&& /* input */,
-    arma::Mat<eT>&& gy,
-    arma::Mat<eT>&& g)
-{
-  g = gy % mask * scale;
-}
+        template<typename InputDataType, typename OutputDataType>
+        template<typename eT>
+        void Dropout<InputDataType, OutputDataType>::Backward(
+                const arma::Mat<eT> && /* input */,
+                arma::Mat<eT> &&gy,
+                arma::Mat<eT> &&g) {
+            g = gy % mask * scale;
+        }
 
-template<typename InputDataType, typename OutputDataType>
-template<typename Archive>
-void Dropout<InputDataType, OutputDataType>::serialize(
-    Archive& ar,
-    const unsigned int /* version */)
-{
-  ar & BOOST_SERIALIZATION_NVP(ratio);
-  ar & BOOST_SERIALIZATION_NVP(rescale);
+        template<typename InputDataType, typename OutputDataType>
+        template<typename Archive>
+        void Dropout<InputDataType, OutputDataType>::serialize(
+                Archive &ar,
+                const unsigned int /* version */) {
+            ar & BOOST_SERIALIZATION_NVP(ratio);
 
-  // Reset scale.
-  scale = 1.0 / (1.0 - ratio);
-}
+            // Reset scale.
+            scale = 1.0 / (1.0 - ratio);
+        }
 
-} // namespace ann
+    } // namespace ann
 } // namespace mlpack
 
 #endif

From 21ba44bb6f81aa7590d360f114069d8f901a2273 Mon Sep 17 00:00:00 2001
From: dakshitagrawal97 <neelagrawal08@gmail.com>
Date: Tue, 27 Feb 2018 07:52:46 +0530
Subject: [PATCH 05/67] fix style errors

---
 src/mlpack/methods/ann/layer/dropout.hpp      | 126 +++++++++---------
 src/mlpack/methods/ann/layer/dropout_impl.hpp |  89 +++++++------
 2 files changed, 110 insertions(+), 105 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/dropout.hpp b/src/mlpack/methods/ann/layer/dropout.hpp
index 55e084bc3b..7af908f2f1 100644
--- a/src/mlpack/methods/ann/layer/dropout.hpp
+++ b/src/mlpack/methods/ann/layer/dropout.hpp
@@ -16,7 +16,8 @@
 #include <mlpack/prereqs.hpp>
 
 namespace mlpack {
-    namespace ann /** Artificial Neural Network. */ {
+namespace ann /** Artificial Neural Network. */ {
+
 
 /**
  * The dropout layer is a regularizer that randomly with probability 'ratio'
@@ -46,80 +47,84 @@ namespace mlpack {
  * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
  */
-        template<
+template<
                 typename InputDataType = arma::mat,
                 typename OutputDataType = arma::mat
         >
-        class Dropout {
-        public:
-            /**
-             * Create the Dropout object using the specified ratio parameter.
-             *
-             * @param ratio The probability of setting a value to zero.
-             */
-            Dropout(const double ratio = 0.5);
+class Dropout {
+ public:
+    /**
+     * Create the Dropout object using the specified ratio parameter.
+     *
+     * @param ratio The probability of setting a value to zero.
+     */
+    Dropout(const double ratio = 0.5);
 
-            /**
-             * Ordinary feed forward pass of the dropout layer.
-             *
-             * @param input Input data used for evaluating the specified function.
-             * @param output Resulting output activation.
-             */
-            template<typename eT>
-            void Forward(const arma::Mat<eT> &&input, arma::Mat<eT> &&output);
+    /**
+     * Ordinary feed forward pass of the dropout layer.
+     *
+     * @param input Input data used for evaluating the specified function.
+     * @param output Resulting output activation.
+     */
+    template<typename eT>
+    void Forward(const arma::Mat<eT> &&input, arma::Mat<eT> &&output);
 
-            /**
-             * Ordinary feed backward pass of the dropout layer.
-             *
-             * @param input The propagated input activation.
-             * @param gy The backpropagated error.
-             * @param g The calculated gradient.
-             */
-            template<typename eT>
-            void Backward(const arma::Mat<eT> && /* input */,
-                          arma::Mat<eT> &&gy,
-                          arma::Mat<eT> &&g);
+    /**
+     * Ordinary feed backward pass of the dropout layer.
+     *
+     * @param input The propagated input activation.
+     * @param gy The backpropagated error.
+     * @param g The calculated gradient.
+     */
+    template<typename eT>
+    void Backward(const arma::Mat<eT> && /* input */,
+                  arma::Mat<eT> &&gy,
+                  arma::Mat<eT> &&g);
 
-            //! Get the input parameter.
-            InputDataType const &InputParameter() const { return inputParameter; }
+    //! Get the input parameter.
+    InputDataType const &InputParameter() const {
+        return inputParameter;
+    }
 
-            //! Modify the input parameter.
-            InputDataType &InputParameter() { return inputParameter; }
+    //! Modify the input parameter.
+    InputDataType &InputParameter() { return inputParameter; }
 
-            //! Get the output parameter.
-            OutputDataType const &OutputParameter() const { return outputParameter; }
+    //! Get the output parameter.
+    OutputDataType const &OutputParameter() const {
+        return outputParameter;
+    }
 
-            //! Modify the output parameter.
-            OutputDataType &OutputParameter() { return outputParameter; }
+    //! Modify the output parameter.
+    OutputDataType &OutputParameter() { return outputParameter; }
 
-            //! Get the detla.
-            OutputDataType const &Delta() const { return delta; }
+    //! Get the detla.
+    OutputDataType const &Delta() const { return delta; }
 
-            //! Modify the delta.
-            OutputDataType &Delta() { return delta; }
+    //! Modify the delta.
+    OutputDataType &Delta() { return delta; }
 
-            //! The value of the deterministic parameter.
-            bool Deterministic() const { return deterministic; }
+    //! The value of the deterministic parameter.
+    bool Deterministic() const { return deterministic; }
 
-            //! Modify the value of the deterministic parameter.
-            bool &Deterministic() { return deterministic; }
+    //! Modify the value of the deterministic parameter.
+    bool &Deterministic() { return deterministic; }
 
-            //! The probability of setting a value to zero.
-            double Ratio() const { return ratio; }
+    //! The probability of setting a value to zero.
+    double Ratio() const { return ratio; }
 
-            //! Modify the probability of setting a value to zero.
-            void Ratio(const double r) {
-                ratio = r;
-                scale = 1.0 / (1.0 - ratio);
-            }
+    //! Modify the probability of setting a value to zero.
+    void Ratio(const double r) {
+        ratio = r;
+        scale = 1.0 / (1.0 - ratio);
+    }
 
-            /**
-             * Serialize the layer.
-             */
-            template<typename Archive>
-            void serialize(Archive &ar, const unsigned int /* version */);
+    /**
+     * Serialize the layer.
+     */
+    template<typename Archive>
+    void serialize(Archive &ar, const unsigned int /* version */);
 
-        private:
+ private:
             //! Locally-stored delta object.
             OutputDataType delta;
 
@@ -140,10 +145,9 @@ namespace mlpack {
 
             //! If true dropout and scaling is disabled, see notes above.
             bool deterministic;
+}; // class Dropout
 
-        }; // class Dropout
-
-    } // namespace ann
+} // namespace ann
 } // namespace mlpack
 
 // Include implementation.
diff --git a/src/mlpack/methods/ann/layer/dropout_impl.hpp b/src/mlpack/methods/ann/layer/dropout_impl.hpp
index 9540fc2cd7..33d591baaf 100644
--- a/src/mlpack/methods/ann/layer/dropout_impl.hpp
+++ b/src/mlpack/methods/ann/layer/dropout_impl.hpp
@@ -17,56 +17,57 @@
 #include "dropout.hpp"
 
 namespace mlpack {
-    namespace ann /** Artificial Neural Network. */ {
+namespace ann /** Artificial Neural Network. */ {
 
-        template<typename InputDataType, typename OutputDataType>
-        Dropout<InputDataType, OutputDataType>::Dropout(
-                const double ratio) :
-                ratio(ratio),
-                scale(1.0 / (1.0 - ratio)),
-                deterministic(false) {
-            // Nothing to do here.
+    template<typename InputDataType, typename OutputDataType>
+    Dropout<InputDataType, OutputDataType>::Dropout(
+            const double ratio) :
+            ratio(ratio),
+            scale(1.0 / (1.0 - ratio)),
+            deterministic(false) {
+        // Nothing to do here.
+    }
+
+    template<typename InputDataType, typename OutputDataType>
+    template<typename eT>
+    void Dropout<InputDataType, OutputDataType>::Forward(
+            const arma::Mat<eT> &&input,
+            arma::Mat<eT> &&output) {
+        // The dropout mask will not be multiplied in the deterministic mode
+        // (during testing).
+        if (deterministic) {
+            output = input;
+        } else {
+            // Scale with input / (1 - ratio) and set values to zero
+            // with probability 'ratio'.
+
+            mask = arma::randu<arma::Mat<eT> >(input.n_rows, input.n_cols);
+            mask.transform([&](double val) { return (val > ratio); });
+            output = input % mask * scale;
         }
+    }
 
-        template<typename InputDataType, typename OutputDataType>
-        template<typename eT>
-        void Dropout<InputDataType, OutputDataType>::Forward(
-                const arma::Mat<eT> &&input,
-                arma::Mat<eT> &&output) {
-            // The dropout mask will not be multiplied in the deterministic mode
-            // (during testing).
-            if (deterministic) {
-                output = input;
-            } else {
-                // Scale with input / (1 - ratio) and set values to zero with probability
-                // ratio.
-                mask = arma::randu < arma::Mat<eT> > (input.n_rows, input.n_cols);
-                mask.transform([&](double val) { return (val > ratio); });
-                output = input % mask * scale;
-            }
-        }
+    template<typename InputDataType, typename OutputDataType>
+    template<typename eT>
+    void Dropout<InputDataType, OutputDataType>::Backward(
+            const arma::Mat<eT> && /* input */,
+            arma::Mat<eT> &&gy,
+            arma::Mat<eT> &&g) {
+        g = gy % mask * scale;
+    }
 
-        template<typename InputDataType, typename OutputDataType>
-        template<typename eT>
-        void Dropout<InputDataType, OutputDataType>::Backward(
-                const arma::Mat<eT> && /* input */,
-                arma::Mat<eT> &&gy,
-                arma::Mat<eT> &&g) {
-            g = gy % mask * scale;
-        }
+    template<typename InputDataType, typename OutputDataType>
+    template<typename Archive>
+    void Dropout<InputDataType, OutputDataType>::serialize(
+            Archive &ar,
+            const unsigned int /* version */) {
+        ar & BOOST_SERIALIZATION_NVP(ratio);
 
-        template<typename InputDataType, typename OutputDataType>
-        template<typename Archive>
-        void Dropout<InputDataType, OutputDataType>::serialize(
-                Archive &ar,
-                const unsigned int /* version */) {
-            ar & BOOST_SERIALIZATION_NVP(ratio);
+        // Reset scale.
+        scale = 1.0 / (1.0 - ratio);
+    }
 
-            // Reset scale.
-            scale = 1.0 / (1.0 - ratio);
-        }
-
-    } // namespace ann
+} // namespace ann
 } // namespace mlpack
 
 #endif

From 298cf109bd68c3df5e376803977afa48a4dcb3c0 Mon Sep 17 00:00:00 2001
From: dakshitagrawal97 <neelagrawal08@gmail.com>
Date: Wed, 28 Feb 2018 20:41:30 +0530
Subject: [PATCH 06/67] fix simple dropout test

---
 src/mlpack/tests/ann_layer_test.cpp | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp
index c768e744d5..206b934157 100644
--- a/src/mlpack/tests/ann_layer_test.cpp
+++ b/src/mlpack/tests/ann_layer_test.cpp
@@ -320,7 +320,6 @@ BOOST_AUTO_TEST_CASE(SimpleDropoutLayerTest)
   // Initialize the probability of setting a value to zero and the scale
   // parameter.
   const double p = 0.2;
-  const double scale = 1.0 / (1.0 - p);
 
   // Initialize the input parameter.
   arma::mat input(1000, 1);
@@ -343,14 +342,8 @@ BOOST_AUTO_TEST_CASE(SimpleDropoutLayerTest)
 
   // Test the Forward function.
   module.Deterministic() = true;
-  module.Rescale() = false;
   module.Forward(std::move(input), std::move(output));
   BOOST_REQUIRE_EQUAL(arma::accu(input), arma::accu(output));
-
-  // Test the Forward function.
-  module.Rescale() = true;
-  module.Forward(std::move(input), std::move(output));
-  BOOST_REQUIRE_CLOSE(arma::accu(input) * scale, arma::accu(output), 1e-3);
 }
 
 /**

From f29ae55bb1e44d315afe1be3da0a3a4069467b10 Mon Sep 17 00:00:00 2001
From: dakshitagrawal97 <neelagrawal08@gmail.com>
Date: Wed, 28 Feb 2018 22:26:51 +0530
Subject: [PATCH 07/67] add He initialization test

---
 src/mlpack/tests/init_rules_test.cpp | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/src/mlpack/tests/init_rules_test.cpp b/src/mlpack/tests/init_rules_test.cpp
index 079c4263e5..c6d1d05936 100644
--- a/src/mlpack/tests/init_rules_test.cpp
+++ b/src/mlpack/tests/init_rules_test.cpp
@@ -23,6 +23,7 @@
 #include <mlpack/methods/ann/init_rules/random_init.hpp>
 #include <mlpack/methods/ann/init_rules/const_init.hpp>
 #include <mlpack/methods/ann/init_rules/gaussian_init.hpp>
+#include <mlpack/methods/ann/init_rules/he_init.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
@@ -197,6 +198,32 @@ BOOST_AUTO_TEST_CASE(GaussianInitTest)
   BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices);
 }
 
+/**
+* Simple test of the HeInitialization class.
+*/
+BOOST_AUTO_TEST_CASE(HeInitTest)
+{
+  const size_t rows = 4;
+  const size_t cols = 4;
+  const size_t slices = 2;
+
+  arma::mat weights;
+  arma::cube weights3d;
+
+  HeInitialization initialization;
+
+  initialization.Initialize(weights, rows, cols);
+  initialization.Initialize(weights3d, rows, cols, slices);
+
+  BOOST_REQUIRE_EQUAL(weights.n_rows, rows);
+  BOOST_REQUIRE_EQUAL(weights.n_cols, cols);
+
+  BOOST_REQUIRE_EQUAL(weights3d.n_rows, rows);
+  BOOST_REQUIRE_EQUAL(weights3d.n_cols, cols);
+  BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices);
+}
+
+
 /**
  * Simple test of the NetworkInitialization class, we test it with every
  * implemented initialization rule and make sure the output is reasonable.

From a4f2ed529ebb4a9e2201e4c6661e6db05e28d12a Mon Sep 17 00:00:00 2001
From: dakshitagrawal97 <neelagrawal08@gmail.com>
Date: Wed, 28 Feb 2018 22:33:21 +0530
Subject: [PATCH 08/67] add lecun normal initialization tests

---
 src/mlpack/tests/init_rules_test.cpp | 32 +++++++++++++++++++++++++---
 1 file changed, 29 insertions(+), 3 deletions(-)

diff --git a/src/mlpack/tests/init_rules_test.cpp b/src/mlpack/tests/init_rules_test.cpp
index c6d1d05936..98dea98c3a 100644
--- a/src/mlpack/tests/init_rules_test.cpp
+++ b/src/mlpack/tests/init_rules_test.cpp
@@ -24,6 +24,7 @@
 #include <mlpack/methods/ann/init_rules/const_init.hpp>
 #include <mlpack/methods/ann/init_rules/gaussian_init.hpp>
 #include <mlpack/methods/ann/init_rules/he_init.hpp>
+#include <mlpack/methods/ann/init_rules/lecun_normal_init.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
@@ -210,10 +211,35 @@ BOOST_AUTO_TEST_CASE(HeInitTest)
   arma::mat weights;
   arma::cube weights3d;
 
-  HeInitialization initialization;
+  HeInitialization initializer;
 
-  initialization.Initialize(weights, rows, cols);
-  initialization.Initialize(weights3d, rows, cols, slices);
+  initializer.Initialize(weights, rows, cols);
+  initializer.Initialize(weights3d, rows, cols, slices);
+
+  BOOST_REQUIRE_EQUAL(weights.n_rows, rows);
+  BOOST_REQUIRE_EQUAL(weights.n_cols, cols);
+
+  BOOST_REQUIRE_EQUAL(weights3d.n_rows, rows);
+  BOOST_REQUIRE_EQUAL(weights3d.n_cols, cols);
+  BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices);
+}
+
+/**
+* Simple test of the LecunNormalInitialization class.
+*/
+BOOST_AUTO_TEST_CASE(LecunNormalInitTest)
+{
+  const size_t rows = 4;
+  const size_t cols = 4;
+  const size_t slices = 2;
+
+  arma::mat weights;
+  arma::cube weights3d;
+
+  LecunNormalInitialization initializer;
+
+  initializer.Initialize(weights, rows, cols);
+  initializer.Initialize(weights3d, rows, cols, slices);
 
   BOOST_REQUIRE_EQUAL(weights.n_rows, rows);
   BOOST_REQUIRE_EQUAL(weights.n_cols, cols);

From 84db5a687fe157052511e2459e08ee63bb980149 Mon Sep 17 00:00:00 2001
From: dakshitagrawal97 <neelagrawal08@gmail.com>
Date: Wed, 28 Feb 2018 22:39:05 +0530
Subject: [PATCH 09/67] fix style errors

---
 src/mlpack/methods/ann/init_rules/he_init.hpp | 109 ++++++++---------
 .../ann/init_rules/lecun_normal_init.hpp      | 115 +++++++++---------
 2 files changed, 111 insertions(+), 113 deletions(-)

diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp
index 58e9732b3b..7f0f7888aa 100644
--- a/src/mlpack/methods/ann/init_rules/he_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/he_init.hpp
@@ -32,69 +32,68 @@
 using namespace mlpack::math;
 
 namespace mlpack {
-    namespace ann /** Artificial Neural Network. */ {
+namespace ann /** Artificial Neural Network. */ {
 
 /**
  * This class is used to initialize weight matrix with the He initialization rule.
  */
-        class HeInitialization
+class HeInitialization
+{
+ public:
+    /**
+     * Initialize the HeInitialization object.
+     *
+     */
+    HeInitialization()
+    {
+        // Nothing to do here.
+    }
+
+    /**
+     * Initialize the elements of the weight matrix with the He initialization
+     * rule.
+     *
+     * @param W Weight matrix to initialize.
+     * @param rows Number of rows.
+     * @param cols Number of columns.
+     */
+    void Initialize(arma::mat& W,
+                    const size_t rows,
+                    const size_t cols)
+    {
+        double_t variance = 2 / rows;
+
+        if (W.is_empty())
         {
-        public:
-            /**
-             * Initialize the HeInitialization object.
-             *
-             */
-            HeInitialization()
-            {
-                // Nothing to do here.
-            }
+            W = arma::mat(rows, cols);
+        }
 
-            /**
-             * Initialize the elements of the weight matrix with the He initialization
-             * rule.
-             *
-             * @param W Weight matrix to initialize.
-             * @param rows Number of rows.
-             * @param cols Number of columns.
-             */
-            void Initialize(arma::mat& W,
-                            const size_t rows,
-                            const size_t cols)
-            {
-                double_t variance = 2 / rows;
+        W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } );
+    }
 
-                if (W.is_empty())
-                {
-                    W = arma::mat(rows, cols);
-                }
+    /**
+     * Initialize the elements of the specified weight 3rd order tensor
+     * with He initialization rule.
+     *
+     * @param W Weight matrix to initialize.
+     * @param rows Number of rows.
+     * @param cols Number of columns.
+     * @param slice Numbers of slices.
+     */
+    void Initialize(arma::cube & W,
+                    const size_t rows,
+                    const size_t cols,
+                    const size_t slices)
+    {
+        W = arma::cube(rows, cols, slices);
 
-                W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } );
-            }
+        for (size_t i = 0; i < slices; i++) {
+            Initialize(W.slice(i), rows, cols);
+        }
+    }
+}; // class HeInitialization
 
-            /**
-             * Initialize the elements of the specified weight 3rd order tensor
-             * with He initialization rule.
-             *
-             * @param W Weight matrix to initialize.
-             * @param rows Number of rows.
-             * @param cols Number of columns.
-             * @param slice Numbers of slices.
-             */
-            void Initialize(arma::cube & W,
-                            const size_t rows,
-                            const size_t cols,
-                            const size_t slices)
-            {
-                W = arma::cube(rows, cols, slices);
-
-                for (size_t i = 0; i < slices; i++) {
-                    Initialize(W.slice(i), rows, cols);
-                }
-            }
-
-        }; // class HeInitialization
-
-    } // namespace ann
+} // namespace ann
 } // namespace mlpack
 
-#endif
\ No newline at end of file
+#endif
diff --git a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
index 14cf63d07d..92bd9b7ee9 100644
--- a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
@@ -36,70 +36,69 @@
 using namespace mlpack::math;
 
 namespace mlpack {
-    namespace ann /** Artificial Neural Network. */ {
+namespace ann /** Artificial Neural Network. */ {
 
 /**
- * This class is used to initialize weight matrix with the Lecun Normalization
- * initialization rule.
- */
-        class LecunNormalInitialization
+* This class is used to initialize weight matrix with the Lecun Normalization
+* initialization rule.
+*/
+class LecunNormalInitialization
+{
+ public:
+    /**
+     * Initialize the LecunNormalInitialization object.
+     *
+     */
+    LecunNormalInitialization()
+    {
+        // Nothing to do here.
+    }
+
+    /**
+     * Initialize the elements of the weight matrix with the Lecun
+     * Normal initialization rule.
+     *
+     * @param W Weight matrix to initialize.
+     * @param rows Number of rows.
+     * @param cols Number of columns.
+     */
+    void Initialize(arma::mat& W,
+                    const size_t rows,
+                    const size_t cols)
+    {
+        double_t variance = 1 / rows;
+
+        if (W.is_empty())
         {
-        public:
-            /**
-             * Initialize the LecunNormalInitialization object.
-             *
-             */
-            LecunNormalInitialization()
-            {
-                // Nothing to do here.
-            }
+            W = arma::mat(rows, cols);
+        }
 
-            /**
-             * Initialize the elements of the weight matrix with the Lecun
-             * Normal initialization rule.
-             *
-             * @param W Weight matrix to initialize.
-             * @param rows Number of rows.
-             * @param cols Number of columns.
-             */
-            void Initialize(arma::mat& W,
-                            const size_t rows,
-                            const size_t cols)
-            {
-                double_t variance = 1 / rows;
+        W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } );
+    }
 
-                if (W.is_empty())
-                {
-                    W = arma::mat(rows, cols);
-                }
+    /**
+     * Initialize the elements of the specified weight 3rd order tensor
+     * with Lecun Normal initialization rule.
+     *
+     * @param W Weight matrix to initialize.
+     * @param rows Number of rows.
+     * @param cols Number of columns.
+     * @param slice Numbers of slices.
+     */
+    void Initialize(arma::cube & W,
+                    const size_t rows,
+                    const size_t cols,
+                    const size_t slices)
+    {
+        W = arma::cube(rows, cols, slices);
 
-                W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } );
-            }
+        for (size_t i = 0; i < slices; i++) {
+            Initialize(W.slice(i), rows, cols);
+        }
+    }
+}; // class LecunNormalInitialization
 
-            /**
-             * Initialize the elements of the specified weight 3rd order tensor
-             * with Lecun Normal initialization rule.
-             *
-             * @param W Weight matrix to initialize.
-             * @param rows Number of rows.
-             * @param cols Number of columns.
-             * @param slice Numbers of slices.
-             */
-            void Initialize(arma::cube & W,
-                            const size_t rows,
-                            const size_t cols,
-                            const size_t slices)
-            {
-                W = arma::cube(rows, cols, slices);
-
-                for (size_t i = 0; i < slices; i++) {
-                    Initialize(W.slice(i), rows, cols);
-                }
-            }
-
-        }; // class LecunNormalInitialization
-
-    } // namespace ann
+} // namespace ann
 } // namespace mlpack
 
-#endif
\ No newline at end of file
+#endif

From 546e6d079965dd82961560007e70277a06b27568 Mon Sep 17 00:00:00 2001
From: dakshitagrawal97 <neelagrawal08@gmail.com>
Date: Thu, 1 Mar 2018 07:30:13 +0530
Subject: [PATCH 10/67] minor changes for performance improvement and
 documentation

---
 src/mlpack/methods/ann/init_rules/he_init.hpp      | 14 +++++++++-----
 .../methods/ann/init_rules/lecun_normal_init.hpp   | 14 +++++++++-----
 2 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp
index 7f0f7888aa..8d569f725b 100644
--- a/src/mlpack/methods/ann/init_rules/he_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/he_init.hpp
@@ -29,8 +29,6 @@
 #include <mlpack/prereqs.hpp>
 #include <mlpack/core/math/random.hpp>
 
-using namespace mlpack::math;
-
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
@@ -61,11 +59,14 @@ class HeInitialization
                     const size_t rows,
                     const size_t cols)
     {
-        double_t variance = 2 / rows;
+        // He initialization rule says to initialize weights with random
+        // values taken from a gaussian distribution with mean = 0 and
+        // standard deviation = sqrt(2/rows), i.e. variance = (2/rows).
+        double_t variance =  2.0 / rows;
 
         if (W.is_empty())
         {
-            W = arma::mat(rows, cols);
+            W.set_size(rows, cols);
         }
 
         W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } );
@@ -85,7 +86,10 @@ class HeInitialization
                     const size_t cols,
                     const size_t slices)
     {
-        W = arma::cube(rows, cols, slices);
+        if (W.is_empty())
+        {
+            W.set_size(rows, cols, slices);
+        }
 
         for (size_t i = 0; i < slices; i++) {
             Initialize(W.slice(i), rows, cols);
diff --git a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
index 92bd9b7ee9..c3778c23a9 100644
--- a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
@@ -33,8 +33,6 @@
 #include <mlpack/prereqs.hpp>
 #include <mlpack/core/math/random.hpp>
 
-using namespace mlpack::math;
-
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
@@ -66,11 +64,14 @@ class LecunNormalInitialization
                     const size_t rows,
                     const size_t cols)
     {
-        double_t variance = 1 / rows;
+        // He initialization rule says to initialize weights with random
+        // values taken from a gaussian distribution with mean = 0 and
+        // standard deviation = sqrt(1/rows), i.e. variance = (1/rows).
+        double_t variance = 1.0 / rows;
 
         if (W.is_empty())
         {
-            W = arma::mat(rows, cols);
+            W.set_size(rows, cols);
         }
 
         W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } );
@@ -90,7 +91,10 @@ class LecunNormalInitialization
                     const size_t cols,
                     const size_t slices)
     {
-        W = arma::cube(rows, cols, slices);
+        if (W.is_empty())
+        {
+            W.set_size(rows, cols, slices);
+        }
 
         for (size_t i = 0; i < slices; i++) {
             Initialize(W.slice(i), rows, cols);

From b033c5acb49e06947364eb1673effff306bc9865 Mon Sep 17 00:00:00 2001
From: dakshitagrawal97 <neelagrawal08@gmail.com>
Date: Thu, 1 Mar 2018 07:32:18 +0530
Subject: [PATCH 11/67] fix type casting error

---
 src/mlpack/methods/ann/init_rules/he_init.hpp           | 2 +-
 src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp
index 8d569f725b..b12dec7d82 100644
--- a/src/mlpack/methods/ann/init_rules/he_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/he_init.hpp
@@ -62,7 +62,7 @@ class HeInitialization
         // He initialization rule says to initialize weights with random
         // values taken from a gaussian distribution with mean = 0 and
         // standard deviation = sqrt(2/rows), i.e. variance = (2/rows).
-        double_t variance =  2.0 / rows;
+        double_t variance =  2.0 / ((double) rows);
 
         if (W.is_empty())
         {
diff --git a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
index c3778c23a9..fac030af9a 100644
--- a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
@@ -67,7 +67,7 @@ class LecunNormalInitialization
         // He initialization rule says to initialize weights with random
         // values taken from a gaussian distribution with mean = 0 and
         // standard deviation = sqrt(1/rows), i.e. variance = (1/rows).
-        double_t variance = 1.0 / rows;
+        double_t variance = 1.0 / ((double) rows);
 
         if (W.is_empty())
         {

From 7fc89f97290571617ce7935786f1df69f19bc88b Mon Sep 17 00:00:00 2001
From: dakshitagrawal97 <neelagrawal08@gmail.com>
Date: Sat, 3 Mar 2018 22:09:21 +0530
Subject: [PATCH 12/67] fixes the indentation and documentation errors

---
 src/mlpack/methods/ann/layer/dropout.hpp      | 138 +++++++++---------
 src/mlpack/methods/ann/layer/dropout_impl.hpp |  88 +++++------
 src/mlpack/tests/ann_layer_test.cpp           |   3 +-
 3 files changed, 114 insertions(+), 115 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/dropout.hpp b/src/mlpack/methods/ann/layer/dropout.hpp
index 7af908f2f1..3614f00680 100644
--- a/src/mlpack/methods/ann/layer/dropout.hpp
+++ b/src/mlpack/methods/ann/layer/dropout.hpp
@@ -53,98 +53,94 @@ template<
         >
 class Dropout {
  public:
-    /**
-     * Create the Dropout object using the specified ratio parameter.
-     *
-     * @param ratio The probability of setting a value to zero.
-     */
-    Dropout(const double ratio = 0.5);
+  /**
+   * Create the Dropout object using the specified ratio parameter.
+   *
+   * @param ratio The probability of setting a value to zero.
+   */
+  Dropout(const double ratio = 0.5);
 
-    /**
-     * Ordinary feed forward pass of the dropout layer.
-     *
-     * @param input Input data used for evaluating the specified function.
-     * @param output Resulting output activation.
-     */
-    template<typename eT>
-    void Forward(const arma::Mat<eT> &&input, arma::Mat<eT> &&output);
+  /**
+   * Ordinary feed forward pass of the dropout layer.
+   *
+   * @param input Input data used for evaluating the specified function.
+   * @param output Resulting output activation.
+   */
+  template<typename eT>
+  void Forward(const arma::Mat<eT> &&input, arma::Mat<eT> &&output);
 
-    /**
-     * Ordinary feed backward pass of the dropout layer.
-     *
-     * @param input The propagated input activation.
-     * @param gy The backpropagated error.
-     * @param g The calculated gradient.
-     */
-    template<typename eT>
-    void Backward(const arma::Mat<eT> && /* input */,
-                  arma::Mat<eT> &&gy,
-                  arma::Mat<eT> &&g);
+  /**
+   * Ordinary feed backward pass of the dropout layer.
+   *
+   * @param input The propagated input activation.
+   * @param gy The backpropagated error.
+   * @param g The calculated gradient.
+   */
+  template<typename eT>
+  void Backward(const arma::Mat<eT> && /* input */,
+                arma::Mat<eT> &&gy,
+                arma::Mat<eT> &&g);
 
-    //! Get the input parameter.
-    InputDataType const &InputParameter() const {
-        return inputParameter;
-    }
+  //! Get the input parameter.
+  InputDataType const &InputParameter() const { return inputParameter;  }
 
-    //! Modify the input parameter.
-    InputDataType &InputParameter() { return inputParameter; }
+  //! Modify the input parameter.
+  InputDataType &InputParameter() { return inputParameter; }
 
-    //! Get the output parameter.
-    OutputDataType const &OutputParameter() const {
-        return outputParameter;
-    }
+  //! Get the output parameter.
+  OutputDataType const &OutputParameter() const { return outputParameter; }
 
-    //! Modify the output parameter.
-    OutputDataType &OutputParameter() { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType &OutputParameter() { return outputParameter; }
 
-    //! Get the detla.
-    OutputDataType const &Delta() const { return delta; }
+  //! Get the detla.
+  OutputDataType const &Delta() const { return delta; }
 
-    //! Modify the delta.
-    OutputDataType &Delta() { return delta; }
+  //! Modify the delta.
+  OutputDataType &Delta() { return delta; }
 
-    //! The value of the deterministic parameter.
-    bool Deterministic() const { return deterministic; }
+  //! The value of the deterministic parameter.
+  bool Deterministic() const { return deterministic; }
 
-    //! Modify the value of the deterministic parameter.
-    bool &Deterministic() { return deterministic; }
+  //! Modify the value of the deterministic parameter.
+  bool &Deterministic() { return deterministic; }
 
-    //! The probability of setting a value to zero.
-    double Ratio() const { return ratio; }
+  //! The probability of setting a value to zero.
+  double Ratio() const { return ratio; }
 
-    //! Modify the probability of setting a value to zero.
-    void Ratio(const double r) {
-        ratio = r;
-        scale = 1.0 / (1.0 - ratio);
-    }
+  //! Modify the probability of setting a value to zero.
+  void Ratio(const double r) {
+    ratio = r;
+    scale = 1.0 / (1.0 - ratio);
+  }
 
-    /**
-     * Serialize the layer.
-     */
-    template<typename Archive>
-    void serialize(Archive &ar, const unsigned int /* version */);
+  /**
+   * Serialize the layer.
+   */
+  template<typename Archive>
+  void serialize(Archive &ar, const unsigned int /* version */);
 
  private:
-            //! Locally-stored delta object.
-            OutputDataType delta;
+  //! Locally-stored delta object.
+  OutputDataType delta;
 
-            //! Locally-stored input parameter object.
-            InputDataType inputParameter;
+  //! Locally-stored input parameter object.
+  InputDataType inputParameter;
 
-            //! Locally-stored output parameter object.
-            OutputDataType outputParameter;
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
 
-            //! Locally-stored mast object.
-            OutputDataType mask;
+  //! Locally-stored mast object.
+  OutputDataType mask;
 
-            //! The probability of setting a value to zero.
-            double ratio;
+  //! The probability of setting a value to zero.
+  double ratio;
 
-            //! The scale fraction.
-            double scale;
+  //! The scale fraction.
+  double scale;
 
-            //! If true dropout and scaling is disabled, see notes above.
-            bool deterministic;
+  //! If true dropout and scaling is disabled, see notes above.
+  bool deterministic;
 }; // class Dropout
 
 } // namespace ann
diff --git a/src/mlpack/methods/ann/layer/dropout_impl.hpp b/src/mlpack/methods/ann/layer/dropout_impl.hpp
index 33d591baaf..42ccfbf47e 100644
--- a/src/mlpack/methods/ann/layer/dropout_impl.hpp
+++ b/src/mlpack/methods/ann/layer/dropout_impl.hpp
@@ -19,53 +19,57 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-    template<typename InputDataType, typename OutputDataType>
-    Dropout<InputDataType, OutputDataType>::Dropout(
-            const double ratio) :
-            ratio(ratio),
-            scale(1.0 / (1.0 - ratio)),
-            deterministic(false) {
-        // Nothing to do here.
-    }
+template<typename InputDataType, typename OutputDataType>
+Dropout<InputDataType, OutputDataType>::Dropout(
+    const double ratio) :
+    ratio(ratio),
+    scale(1.0 / (1.0 - ratio)),
+    deterministic(false)
+{
+  // Nothing to do here.
+}
 
-    template<typename InputDataType, typename OutputDataType>
-    template<typename eT>
-    void Dropout<InputDataType, OutputDataType>::Forward(
-            const arma::Mat<eT> &&input,
-            arma::Mat<eT> &&output) {
-        // The dropout mask will not be multiplied in the deterministic mode
-        // (during testing).
-        if (deterministic) {
-            output = input;
-        } else {
-            // Scale with input / (1 - ratio) and set values to zero
-            // with probability 'ratio'.
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void Dropout<InputDataType, OutputDataType>::Forward(
+    const arma::Mat<eT> &&input,
+    arma::Mat<eT> &&output)
+{
+  // The dropout mask will not be multiplied in the deterministic mode
+  // (during testing).
+  if (deterministic) {
+    output = input;
+  } else {
+    // Scale with input / (1 - ratio) and set values to zero
+    // with probability 'ratio'.
 
-            mask = arma::randu<arma::Mat<eT> >(input.n_rows, input.n_cols);
-            mask.transform([&](double val) { return (val > ratio); });
-            output = input % mask * scale;
-        }
-    }
+    mask = arma::randu<arma::Mat<eT> >(input.n_rows, input.n_cols);
+    mask.transform([&](double val) { return (val > ratio); });
+    output = input % mask * scale;
+  }
+}
 
-    template<typename InputDataType, typename OutputDataType>
-    template<typename eT>
-    void Dropout<InputDataType, OutputDataType>::Backward(
-            const arma::Mat<eT> && /* input */,
-            arma::Mat<eT> &&gy,
-            arma::Mat<eT> &&g) {
-        g = gy % mask * scale;
-    }
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void Dropout<InputDataType, OutputDataType>::Backward(
+    const arma::Mat<eT> && /* input */,
+    arma::Mat<eT> &&gy,
+    arma::Mat<eT> &&g)
+{
+  g = gy % mask * scale;
+}
 
-    template<typename InputDataType, typename OutputDataType>
-    template<typename Archive>
-    void Dropout<InputDataType, OutputDataType>::serialize(
-            Archive &ar,
-            const unsigned int /* version */) {
-        ar & BOOST_SERIALIZATION_NVP(ratio);
+template<typename InputDataType, typename OutputDataType>
+template<typename Archive>
+void Dropout<InputDataType, OutputDataType>::serialize(
+    Archive &ar,
+    const unsigned int /* version */)
+{
+  ar & BOOST_SERIALIZATION_NVP(ratio);
 
-        // Reset scale.
-        scale = 1.0 / (1.0 - ratio);
-    }
+  // Reset scale.
+  scale = 1.0 / (1.0 - ratio);
+}
 
 } // namespace ann
 } // namespace mlpack
diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp
index 206b934157..14b9637c52 100644
--- a/src/mlpack/tests/ann_layer_test.cpp
+++ b/src/mlpack/tests/ann_layer_test.cpp
@@ -317,8 +317,7 @@ BOOST_AUTO_TEST_CASE(JacobianConstantLayerTest)
  */
 BOOST_AUTO_TEST_CASE(SimpleDropoutLayerTest)
 {
-  // Initialize the probability of setting a value to zero and the scale
-  // parameter.
+  // Initialize the probability of setting a value to zero.
   const double p = 0.2;
 
   // Initialize the input parameter.

From b5778d00a969c074ed76738571b11227d52868e8 Mon Sep 17 00:00:00 2001
From: dakshitagrawal97 <neelagrawal08@gmail.com>
Date: Sat, 3 Mar 2018 23:20:54 +0530
Subject: [PATCH 13/67] make changes in documentation and code

---
 src/mlpack/methods/ann/init_rules/he_init.hpp | 23 ++++++++++++++++---
 .../ann/init_rules/lecun_normal_init.hpp      | 23 +++++++++++++++++--
 2 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp
index b12dec7d82..7b96fcb379 100644
--- a/src/mlpack/methods/ann/init_rules/he_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/he_init.hpp
@@ -33,7 +33,22 @@ namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
 /**
- * This class is used to initialize weight matrix with the He initialization rule.
+ * This class is used to initialize weight matrix with the He
+ * initialization rule given by He et. al. for neural networks. The He
+ * initialization initializes weights of the neural network to better
+ * suit the rectified activation units.
+ *
+ * For more information, the following paper can be referred to:
+ *
+ * @code
+ * @article{He2015DelvingDI,
+ * title={Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification},
+ * author={Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun},
+ * journal={2015 IEEE International Conference on Computer Vision (ICCV)},
+ * year={2015},
+ * pages={1026-1034}}
+ * @endcode
+ *
  */
 class HeInitialization
 {
@@ -62,14 +77,16 @@ class HeInitialization
         // He initialization rule says to initialize weights with random
         // values taken from a gaussian distribution with mean = 0 and
         // standard deviation = sqrt(2/rows), i.e. variance = (2/rows).
-        double_t variance =  2.0 / ((double) rows);
+        double variance =  2.0 / ((double) rows);
 
         if (W.is_empty())
         {
             W.set_size(rows, cols);
         }
 
-        W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } );
+        // Multipling a random variable X with variance V(X) by some factor c,
+        // then the variance V(cX) = (c^2)* V(X).
+        W.imbue( [&]() { return sqrt(variance) * arma::randn();} );
     }
 
     /**
diff --git a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
index fac030af9a..27c5205fbe 100644
--- a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
@@ -39,6 +39,23 @@ namespace ann /** Artificial Neural Network. */ {
 /**
 * This class is used to initialize weight matrix with the Lecun Normalization
 * initialization rule.
+ *
+ * For more information, the following papers can be referred to:
+ *
+ * @code
+ * @inproceedings{conf/nips/KlambauerUMH17,
+ * title = {Self-Normalizing Neural Networks.},
+ * author = {Klambauer, Günter and Unterthiner, Thomas and Mayr, Andreas and Hochreiter, Sepp},
+ * pages = {972-981},
+ * year = 2017}
+ *
+ * @inproceedings{LeCun:1998:EB:645754.668382,
+ * title = {Efficient BackProp},
+ * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B. and M\"{u}ller, Klaus-Robert},
+ * year = {1998},
+ * pages = {9--50}}
+ * @endcode
+ *
 */
 class LecunNormalInitialization
 {
@@ -67,14 +84,16 @@ class LecunNormalInitialization
         // He initialization rule says to initialize weights with random
         // values taken from a gaussian distribution with mean = 0 and
         // standard deviation = sqrt(1/rows), i.e. variance = (1/rows).
-        double_t variance = 1.0 / ((double) rows);
+        double variance = 1.0 / ((double) rows);
 
         if (W.is_empty())
         {
             W.set_size(rows, cols);
         }
 
-        W.imbue( [&]() { return arma::as_scalar(RandNormal(0, variance)); } );
+        // Multipling a random variable X with variance V(X) by some factor c,
+        // then the variance V(cX) = (c^2)* V(X).
+        W.imbue( [&]() { return sqrt(variance) * arma::randn();} );
     }
 
     /**

From 97de396c680bb3ed9ae1eb99b4841476a6845d2c Mon Sep 17 00:00:00 2001
From: dakshitagrawal97 <neelagrawal08@gmail.com>
Date: Sat, 3 Mar 2018 23:29:43 +0530
Subject: [PATCH 14/67] fix indentation error

---
 src/mlpack/methods/ann/init_rules/he_init.hpp | 110 ++++++++--------
 .../ann/init_rules/lecun_normal_init.hpp      | 120 +++++++++---------
 2 files changed, 118 insertions(+), 112 deletions(-)

diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp
index 7b96fcb379..f0f38202f6 100644
--- a/src/mlpack/methods/ann/init_rules/he_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/he_init.hpp
@@ -10,7 +10,8 @@
  *
  * @code
  * @article{He2015DelvingDI,
- * title={Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification},
+ * title={Delving Deep into Rectifiers: Surpassing Human-Level Performance
+ * on ImageNet Classification},
  * author={Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun},
  * journal={2015 IEEE International Conference on Computer Vision (ICCV)},
  * year={2015},
@@ -42,7 +43,8 @@ namespace ann /** Artificial Neural Network. */ {
  *
  * @code
  * @article{He2015DelvingDI,
- * title={Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification},
+ * title={Delving Deep into Rectifiers: Surpassing Human-Level Performance
+ * on ImageNet Classification},
  * author={Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun},
  * journal={2015 IEEE International Conference on Computer Vision (ICCV)},
  * year={2015},
@@ -53,65 +55,65 @@ namespace ann /** Artificial Neural Network. */ {
 class HeInitialization
 {
  public:
-    /**
-     * Initialize the HeInitialization object.
-     *
-     */
-    HeInitialization()
+  /**
+   * Initialize the HeInitialization object.
+   *
+   */
+  HeInitialization()
+  {
+    // Nothing to do here.
+  }
+
+  /**
+   * Initialize the elements of the weight matrix with the He initialization
+   * rule.
+   *
+   * @param W Weight matrix to initialize.
+   * @param rows Number of rows.
+   * @param cols Number of columns.
+   */
+  void Initialize(arma::mat& W,
+                  const size_t rows,
+                  const size_t cols)
+  {
+    // He initialization rule says to initialize weights with random
+    // values taken from a gaussian distribution with mean = 0 and
+    // standard deviation = sqrt(2/rows), i.e. variance = (2/rows).
+    double variance =  2.0 / ((double) rows);
+
+    if (W.is_empty())
     {
-        // Nothing to do here.
+      W.set_size(rows, cols);
     }
 
-    /**
-     * Initialize the elements of the weight matrix with the He initialization
-     * rule.
-     *
-     * @param W Weight matrix to initialize.
-     * @param rows Number of rows.
-     * @param cols Number of columns.
-     */
-    void Initialize(arma::mat& W,
-                    const size_t rows,
-                    const size_t cols)
+    // Multipling a random variable X with variance V(X) by some factor c,
+    // then the variance V(cX) = (c^2)* V(X).
+    W.imbue( [&]() { return sqrt(variance) * arma::randn();} );
+  }
+
+  /**
+   * Initialize the elements of the specified weight 3rd order tensor
+   * with He initialization rule.
+   *
+   * @param W Weight matrix to initialize.
+   * @param rows Number of rows.
+   * @param cols Number of columns.
+   * @param slice Numbers of slices.
+   */
+  void Initialize(arma::cube & W,
+                  const size_t rows,
+                  const size_t cols,
+                  const size_t slices)
+  {
+    if (W.is_empty())
     {
-        // He initialization rule says to initialize weights with random
-        // values taken from a gaussian distribution with mean = 0 and
-        // standard deviation = sqrt(2/rows), i.e. variance = (2/rows).
-        double variance =  2.0 / ((double) rows);
-
-        if (W.is_empty())
-        {
-            W.set_size(rows, cols);
-        }
-
-        // Multipling a random variable X with variance V(X) by some factor c,
-        // then the variance V(cX) = (c^2)* V(X).
-        W.imbue( [&]() { return sqrt(variance) * arma::randn();} );
+      W.set_size(rows, cols, slices);
     }
 
-    /**
-     * Initialize the elements of the specified weight 3rd order tensor
-     * with He initialization rule.
-     *
-     * @param W Weight matrix to initialize.
-     * @param rows Number of rows.
-     * @param cols Number of columns.
-     * @param slice Numbers of slices.
-     */
-    void Initialize(arma::cube & W,
-                    const size_t rows,
-                    const size_t cols,
-                    const size_t slices)
-    {
-        if (W.is_empty())
-        {
-            W.set_size(rows, cols, slices);
-        }
-
-        for (size_t i = 0; i < slices; i++) {
-            Initialize(W.slice(i), rows, cols);
-        }
+    for (size_t i = 0; i < slices; i++) {
+      Initialize(W.slice(i), rows, cols);
     }
+  }
 }; // class HeInitialization
 
 } // namespace ann
diff --git a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
index 27c5205fbe..60d3f97bd4 100644
--- a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
@@ -10,13 +10,15 @@
  * @code
  * @inproceedings{conf/nips/KlambauerUMH17,
  * title = {Self-Normalizing Neural Networks.},
- * author = {Klambauer, Günter and Unterthiner, Thomas and Mayr, Andreas and Hochreiter, Sepp},
+ * author = {Klambauer, Günter and Unterthiner, Thomas
+ * and Mayr, Andreas and Hochreiter, Sepp},
  * pages = {972-981},
  * year = 2017}
  *
  * @inproceedings{LeCun:1998:EB:645754.668382,
  * title = {Efficient BackProp},
- * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B. and M\"{u}ller, Klaus-Robert},
+ * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B.
+ * and M\"{u}ller, Klaus-Robert},
  * year = {1998},
  * pages = {9--50}}
  * @endcode
@@ -37,21 +39,23 @@ namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
 /**
-* This class is used to initialize weight matrix with the Lecun Normalization
-* initialization rule.
+ * This class is used to initialize weight matrix with the Lecun Normalization
+ * initialization rule.
  *
  * For more information, the following papers can be referred to:
  *
  * @code
  * @inproceedings{conf/nips/KlambauerUMH17,
  * title = {Self-Normalizing Neural Networks.},
- * author = {Klambauer, Günter and Unterthiner, Thomas and Mayr, Andreas and Hochreiter, Sepp},
+ * author = {Klambauer, Günter and Unterthiner, Thomas
+ * and Mayr, Andreas and Hochreiter, Sepp},
  * pages = {972-981},
  * year = 2017}
  *
  * @inproceedings{LeCun:1998:EB:645754.668382,
  * title = {Efficient BackProp},
- * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B. and M\"{u}ller, Klaus-Robert},
+ * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B.
+ * and M\"{u}ller, Klaus-Robert},
  * year = {1998},
  * pages = {9--50}}
  * @endcode
@@ -60,65 +64,65 @@ namespace ann /** Artificial Neural Network. */ {
 class LecunNormalInitialization
 {
  public:
-    /**
-     * Initialize the LecunNormalInitialization object.
-     *
-     */
-    LecunNormalInitialization()
+  /**
+   * Initialize the LecunNormalInitialization object.
+   *
+   */
+  LecunNormalInitialization()
+  {
+    // Nothing to do here.
+  }
+
+  /**
+   * Initialize the elements of the weight matrix with the Lecun
+   * Normal initialization rule.
+   *
+   * @param W Weight matrix to initialize.
+   * @param rows Number of rows.
+   * @param cols Number of columns.
+   */
+  void Initialize(arma::mat& W,
+                  const size_t rows,
+                  const size_t cols)
+  {
+    // He initialization rule says to initialize weights with random
+    // values taken from a gaussian distribution with mean = 0 and
+    // standard deviation = sqrt(1/rows), i.e. variance = (1/rows).
+    double variance = 1.0 / ((double) rows);
+
+    if (W.is_empty())
     {
-        // Nothing to do here.
+      W.set_size(rows, cols);
     }
 
-    /**
-     * Initialize the elements of the weight matrix with the Lecun
-     * Normal initialization rule.
-     *
-     * @param W Weight matrix to initialize.
-     * @param rows Number of rows.
-     * @param cols Number of columns.
-     */
-    void Initialize(arma::mat& W,
-                    const size_t rows,
-                    const size_t cols)
+    // Multipling a random variable X with variance V(X) by some factor c,
+    // then the variance V(cX) = (c^2)* V(X).
+    W.imbue( [&]() { return sqrt(variance) * arma::randn();} );
+  }
+
+  /**
+   * Initialize the elements of the specified weight 3rd order tensor
+   * with Lecun Normal initialization rule.
+   *
+   * @param W Weight matrix to initialize.
+   * @param rows Number of rows.
+   * @param cols Number of columns.
+   * @param slice Numbers of slices.
+   */
+  void Initialize(arma::cube & W,
+                  const size_t rows,
+                  const size_t cols,
+                  const size_t slices)
+  {
+    if (W.is_empty())
     {
-        // He initialization rule says to initialize weights with random
-        // values taken from a gaussian distribution with mean = 0 and
-        // standard deviation = sqrt(1/rows), i.e. variance = (1/rows).
-        double variance = 1.0 / ((double) rows);
-
-        if (W.is_empty())
-        {
-            W.set_size(rows, cols);
-        }
-
-        // Multipling a random variable X with variance V(X) by some factor c,
-        // then the variance V(cX) = (c^2)* V(X).
-        W.imbue( [&]() { return sqrt(variance) * arma::randn();} );
+      W.set_size(rows, cols, slices);
     }
 
-    /**
-     * Initialize the elements of the specified weight 3rd order tensor
-     * with Lecun Normal initialization rule.
-     *
-     * @param W Weight matrix to initialize.
-     * @param rows Number of rows.
-     * @param cols Number of columns.
-     * @param slice Numbers of slices.
-     */
-    void Initialize(arma::cube & W,
-                    const size_t rows,
-                    const size_t cols,
-                    const size_t slices)
-    {
-        if (W.is_empty())
-        {
-            W.set_size(rows, cols, slices);
-        }
-
-        for (size_t i = 0; i < slices; i++) {
-            Initialize(W.slice(i), rows, cols);
-        }
+    for (size_t i = 0; i < slices; i++) {
+      Initialize(W.slice(i), rows, cols);
     }
+  }
 }; // class LecunNormalInitialization
 
 } // namespace ann

From d0839d12586ee7283c7d99c29125f2fae68a870f Mon Sep 17 00:00:00 2001
From: Prabhat <prabhatsharma7298@gmail.com>
Date: Fri, 30 Mar 2018 21:21:22 +0530
Subject: [PATCH 15/67] Minor style changes

---
 src/mlpack/methods/ann/init_rules/he_init.hpp | 23 +++------------
 .../ann/init_rules/lecun_normal_init.hpp      | 29 +++----------------
 src/mlpack/tests/init_rules_test.cpp          |  1 -
 3 files changed, 8 insertions(+), 45 deletions(-)

diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp
index f0f38202f6..b0ee3c62b7 100644
--- a/src/mlpack/methods/ann/init_rules/he_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/he_init.hpp
@@ -1,23 +1,11 @@
 /**
  * @file he_init.hpp
- * @author Dakshit Agrawal
+ * @authors Dakshit Agrawal and Prabhat Sharma
  *
  * Intialization rule given by He et. al. for neural networks. The He
  * initialization initializes weights of the neural network to better
  * suit the rectified activation units.
  *
- * For more information, the following paper can be referred to:
- *
- * @code
- * @article{He2015DelvingDI,
- * title={Delving Deep into Rectifiers: Surpassing Human-Level Performance
- * on ImageNet Classification},
- * author={Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun},
- * journal={2015 IEEE International Conference on Computer Vision (ICCV)},
- * year={2015},
- * pages={1026-1034}}
- * @endcode
- *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
  * 3-clause BSD license along with mlpack.  If not, see
@@ -79,7 +67,7 @@ class HeInitialization
     // He initialization rule says to initialize weights with random
     // values taken from a gaussian distribution with mean = 0 and
     // standard deviation = sqrt(2/rows), i.e. variance = (2/rows).
-    double variance =  2.0 / ((double) rows);
+    const double variance = 2.0 / (double)rows;
 
     if (W.is_empty())
     {
@@ -88,7 +76,7 @@ class HeInitialization
 
     // Multipling a random variable X with variance V(X) by some factor c,
     // then the variance V(cX) = (c^2)* V(X).
-    W.imbue( [&]() { return sqrt(variance) * arma::randn();} );
+    W.imbue( [&]() { return sqrt(variance) * arma::randn(); } );
   }
 
   /**
@@ -106,13 +94,10 @@ class HeInitialization
                   const size_t slices)
   {
     if (W.is_empty())
-    {
       W.set_size(rows, cols, slices);
-    }
 
-    for (size_t i = 0; i < slices; i++) {
+    for (size_t i = 0; i < slices; i++)
       Initialize(W.slice(i), rows, cols);
-    }
   }
 }; // class HeInitialization
 
diff --git a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
index 60d3f97bd4..4a1bc9ce04 100644
--- a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
@@ -1,28 +1,10 @@
 /**
  * @file lecun_normal_init.hpp
- * @author Dakshit Agrawal
+ * @author Dakshit Agrawal and Prabhat Sharma
  *
  * Intialization rule given by Lecun et. al. for neural networks and
  * also mentioned in Self Normalizing Networks.
  *
- * For more information, the following papers can be referred to:
- *
- * @code
- * @inproceedings{conf/nips/KlambauerUMH17,
- * title = {Self-Normalizing Neural Networks.},
- * author = {Klambauer, Günter and Unterthiner, Thomas
- * and Mayr, Andreas and Hochreiter, Sepp},
- * pages = {972-981},
- * year = 2017}
- *
- * @inproceedings{LeCun:1998:EB:645754.668382,
- * title = {Efficient BackProp},
- * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B.
- * and M\"{u}ller, Klaus-Robert},
- * year = {1998},
- * pages = {9--50}}
- * @endcode
- *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
  * 3-clause BSD license along with mlpack.  If not, see
@@ -88,7 +70,7 @@ class LecunNormalInitialization
     // He initialization rule says to initialize weights with random
     // values taken from a gaussian distribution with mean = 0 and
     // standard deviation = sqrt(1/rows), i.e. variance = (1/rows).
-    double variance = 1.0 / ((double) rows);
+    const double variance = 1.0 / ((double) rows);
 
     if (W.is_empty())
     {
@@ -97,7 +79,7 @@ class LecunNormalInitialization
 
     // Multipling a random variable X with variance V(X) by some factor c,
     // then the variance V(cX) = (c^2)* V(X).
-    W.imbue( [&]() { return sqrt(variance) * arma::randn();} );
+    W.imbue( [&]() { return sqrt(variance) * arma::randn(); } );
   }
 
   /**
@@ -115,13 +97,10 @@ class LecunNormalInitialization
                   const size_t slices)
   {
     if (W.is_empty())
-    {
       W.set_size(rows, cols, slices);
-    }
 
-    for (size_t i = 0; i < slices; i++) {
+    for (size_t i = 0; i < slices; i++)
       Initialize(W.slice(i), rows, cols);
-    }
   }
 }; // class LecunNormalInitialization
 
diff --git a/src/mlpack/tests/init_rules_test.cpp b/src/mlpack/tests/init_rules_test.cpp
index 98dea98c3a..5ebc180eaf 100644
--- a/src/mlpack/tests/init_rules_test.cpp
+++ b/src/mlpack/tests/init_rules_test.cpp
@@ -249,7 +249,6 @@ BOOST_AUTO_TEST_CASE(LecunNormalInitTest)
   BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices);
 }
 
-
 /**
  * Simple test of the NetworkInitialization class, we test it with every
  * implemented initialization rule and make sure the output is reasonable.

From 2727a78454690354ec7deff8f93b326bb3c8e930 Mon Sep 17 00:00:00 2001
From: Prabhat <prabhatsharma7298@gmail.com>
Date: Sat, 31 Mar 2018 03:43:54 +0530
Subject: [PATCH 16/67] Order of tests changed

---
 src/mlpack/tests/init_rules_test.cpp | 99 ++++++++++++++--------------
 1 file changed, 49 insertions(+), 50 deletions(-)

diff --git a/src/mlpack/tests/init_rules_test.cpp b/src/mlpack/tests/init_rules_test.cpp
index 93d1d57205..5d2677b26a 100644
--- a/src/mlpack/tests/init_rules_test.cpp
+++ b/src/mlpack/tests/init_rules_test.cpp
@@ -200,56 +200,6 @@ BOOST_AUTO_TEST_CASE(GaussianInitTest)
   BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices);
 }
 
-/**
-* Simple test of the HeInitialization class.
-*/
-BOOST_AUTO_TEST_CASE(HeInitTest)
-{
-  const size_t rows = 4;
-  const size_t cols = 4;
-  const size_t slices = 2;
-
-  arma::mat weights;
-  arma::cube weights3d;
-
-  HeInitialization initializer;
-
-  initializer.Initialize(weights, rows, cols);
-  initializer.Initialize(weights3d, rows, cols, slices);
-
-  BOOST_REQUIRE_EQUAL(weights.n_rows, rows);
-  BOOST_REQUIRE_EQUAL(weights.n_cols, cols);
-
-  BOOST_REQUIRE_EQUAL(weights3d.n_rows, rows);
-  BOOST_REQUIRE_EQUAL(weights3d.n_cols, cols);
-  BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices);
-}
-
-/**
-* Simple test of the LecunNormalInitialization class.
-*/
-BOOST_AUTO_TEST_CASE(LecunNormalInitTest)
-{
-  const size_t rows = 4;
-  const size_t cols = 4;
-  const size_t slices = 2;
-
-  arma::mat weights;
-  arma::cube weights3d;
-
-  LecunNormalInitialization initializer;
-
-  initializer.Initialize(weights, rows, cols);
-  initializer.Initialize(weights3d, rows, cols, slices);
-
-  BOOST_REQUIRE_EQUAL(weights.n_rows, rows);
-  BOOST_REQUIRE_EQUAL(weights.n_cols, cols);
-
-  BOOST_REQUIRE_EQUAL(weights3d.n_rows, rows);
-  BOOST_REQUIRE_EQUAL(weights3d.n_cols, cols);
-  BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices);
-}
-
 /**
  * Simple test of the NetworkInitialization class, we test it with every
  * implemented initialization rule and make sure the output is reasonable.
@@ -380,5 +330,54 @@ BOOST_AUTO_TEST_CASE(GlorotInitNormalTest)
   BOOST_REQUIRE_EQUAL(weights3d.n_slices, 2);
 }
 
+/**
+* Simple test of the HeInitialization class.
+*/
+BOOST_AUTO_TEST_CASE(HeInitTest)
+{
+  const size_t rows = 4;
+  const size_t cols = 4;
+  const size_t slices = 2;
+
+  arma::mat weights;
+  arma::cube weights3d;
+
+  HeInitialization initializer;
+
+  initializer.Initialize(weights, rows, cols);
+  initializer.Initialize(weights3d, rows, cols, slices);
+
+  BOOST_REQUIRE_EQUAL(weights.n_rows, rows);
+  BOOST_REQUIRE_EQUAL(weights.n_cols, cols);
+
+  BOOST_REQUIRE_EQUAL(weights3d.n_rows, rows);
+  BOOST_REQUIRE_EQUAL(weights3d.n_cols, cols);
+  BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices);
+}
+
+/**
+* Simple test of the LecunNormalInitialization class.
+*/
+BOOST_AUTO_TEST_CASE(LecunNormalInitTest)
+{
+  const size_t rows = 4;
+  const size_t cols = 4;
+  const size_t slices = 2;
+
+  arma::mat weights;
+  arma::cube weights3d;
+
+  LecunNormalInitialization initializer;
+
+  initializer.Initialize(weights, rows, cols);
+  initializer.Initialize(weights3d, rows, cols, slices);
+
+  BOOST_REQUIRE_EQUAL(weights.n_rows, rows);
+  BOOST_REQUIRE_EQUAL(weights.n_cols, cols);
+
+  BOOST_REQUIRE_EQUAL(weights3d.n_rows, rows);
+  BOOST_REQUIRE_EQUAL(weights3d.n_cols, cols);
+  BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices);
+}
 
 BOOST_AUTO_TEST_SUITE_END();

From 95763a7246c9fe6566ee7d14cd6c853b4325e123 Mon Sep 17 00:00:00 2001
From: Projyal Dev <projyal@gmail.com>
Date: Sat, 31 Mar 2018 20:18:42 +0530
Subject: [PATCH 17/67] Changes to accept one output for multiple timesteps

---
 src/mlpack/methods/ann/rnn_impl.hpp | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/methods/ann/rnn_impl.hpp b/src/mlpack/methods/ann/rnn_impl.hpp
index 1dd90725b4..33db24718e 100644
--- a/src/mlpack/methods/ann/rnn_impl.hpp
+++ b/src/mlpack/methods/ann/rnn_impl.hpp
@@ -227,6 +227,7 @@ double RNN<OutputLayerType, InitializationRuleType, CustomLayers...>::Evaluate(
   ResetCells();
 
   double performance = 0;
+  size_t respSeq = 0;
 
   for (size_t seqNum = 0; seqNum < rho; ++seqNum)
   {
@@ -234,7 +235,11 @@ double RNN<OutputLayerType, InitializationRuleType, CustomLayers...>::Evaluate(
     arma::mat stepData(predictors.slice(seqNum).colptr(begin),
         predictors.n_rows, batchSize, false, true);
     Forward(std::move(stepData));
-    arma::mat respData(responses.slice(seqNum).colptr(begin),
+    if (!single)
+    {
+      respSeq = seqNum;
+    }
+    arma::mat respData(responses.slice(respSeq).colptr(begin),
         responses.n_rows, batchSize, false, true);
 
     if (!deterministic)
@@ -248,7 +253,7 @@ double RNN<OutputLayerType, InitializationRuleType, CustomLayers...>::Evaluate(
 
     performance += outputLayer.Forward(std::move(boost::apply_visitor(
         outputParameterVisitor, network.back())),
-        std::move(arma::mat(responses.slice(seqNum).colptr(begin),
+        std::move(arma::mat(responses.slice(respSeq).colptr(begin),
             responses.n_rows, batchSize, false, true)));
   }
 
@@ -309,6 +314,14 @@ void RNN<OutputLayerType, InitializationRuleType, CustomLayers...>::Gradient(
     {
       error.zeros();
     }
+    else if (single && seqNum == 0)
+    {
+      outputLayer.Backward(std::move(boost::apply_visitor(
+          outputParameterVisitor, network.back())),
+          std::move(arma::mat(responses.slice(0).colptr(begin),
+              responses.n_rows, batchSize, false, true)),
+          std::move(error));
+    }
     else
     {
       outputLayer.Backward(std::move(boost::apply_visitor(

From 3d07d99a5d5fcd3a3f75581f5edd408b8e8eb8d2 Mon Sep 17 00:00:00 2001
From: Projyal Dev <projyal@gmail.com>
Date: Sun, 1 Apr 2018 07:24:18 +0530
Subject: [PATCH 18/67] Added tests for RNN with multiple timestep input and
 single output.

---
 src/mlpack/tests/recurrent_network_test.cpp | 140 ++++++++++++++++++++
 1 file changed, 140 insertions(+)

diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp
index 3daa866225..35bb7c95b3 100644
--- a/src/mlpack/tests/recurrent_network_test.cpp
+++ b/src/mlpack/tests/recurrent_network_test.cpp
@@ -15,6 +15,9 @@
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/ann/rnn.hpp>
 #include <mlpack/core/data/binarize.hpp>
+#include <cmath>
+#include <random>
+#include <chrono>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
@@ -1085,4 +1088,141 @@ BOOST_AUTO_TEST_CASE(CustomRecursiveReberGrammarTest)
   ReberGrammarTestCustomNetwork(16, true);
 }
 
+/**
+ * @brief Generates noisy sine wave.
+ *
+ * @param data    The data points as output
+ * @param labels  The expected values as output
+ * @param rho     The size of the sequence of each data point
+ * @param outputSteps How many output steps to consider
+ * @param dataPoints  The number of input data points. The actual generated data
+ * points may be more than this to adjust to the outputSteps. But at the minimum
+ * these many data points will be generated.
+ * @param gain    The gain on the amplitude
+ * @param freq    The frquency of the sine wave
+ * @param phase   The phase shift if any
+ * @param noisePercent  The percent noise to induce
+ * @param numCycles  How many full size wave cycles required. All the data
+ * points will be fit into these cycles.
+ * @param normalize Whether to normalise the data. This may be required for some
+ * layers like LSTM. Default is true.
+ */
+void GenerateNoisySinRNN(arma::cube& data, arma::cube& labels, size_t rho,
+                         size_t outputSteps = 1, const int dataPoints = 100,
+                         const double gain = 1.0, const int freq = 10,
+                         const double phase = 0, const int noisePercent = 20,
+                         const double numCycles = 6.0,
+                         const bool   normalize = true)
+{
+  int points = dataPoints;
+  int r = dataPoints % rho;
+  if (r == 0)
+  {
+    points += outputSteps;
+  }
+  else
+  {
+    points += rho - r + outputSteps;
+  }
+  arma::colvec    x(points);
+  int             i = 0;
+  double          interval = numCycles / freq / points;
+  std::mt19937_64 rng;
+  // initialize the random number generator with time-dependent seed
+  uint64_t timeSeed =
+      std::chrono::high_resolution_clock::now().time_since_epoch().count();
+  std::seed_seq ss{uint32_t(timeSeed & 0xffffffff), uint32_t(timeSeed >> 32)};
+  rng.seed(ss);
+  // initialize a uniform distribution between 0 and 1
+  std::uniform_real_distribution< double > unif(0, 1);
+  x.for_each([&i, gain, freq, phase, noisePercent, interval, &rng,
+              &unif](arma::colvec::elem_type& val) {
+    double t = interval * (i++);
+    val = gain * ::sin(2 * M_PI * freq * t + phase) +
+          (noisePercent * gain / 100 * unif(rng));
+  });
+
+  arma::colvec y = x;
+  if (normalize)
+    y = arma::normalise(x);
+  y.save("rawsin.csv", arma::csv_ascii);
+
+  // Now break this into columns of rho size slices.
+  size_t n_columns = y.n_elem / rho;
+  data = arma::cube(1, n_columns, rho);
+  labels = arma::cube(outputSteps, n_columns, 1);
+  for (int i = 0; i < n_columns; ++i)
+  {
+    data.tube(0, i) = y.rows(i * rho, i * rho + rho - 1);
+    labels.subcube(0, i, 0, outputSteps - 1, i, 0) =
+        y.rows(i * rho + rho, i * rho + rho + outputSteps - 1);
+  }
+}
+
+/**
+ * @brief RNNSineTest Test a simple RNN using noisy sine. Use single output
+ *        for multiple inputs.
+ * @param hiddenUnits No of units in the hiddenlayer.
+ * @param rho The input sequence length.
+ * @param numEpochs The number of epochs to run.
+ * @return The mean squared error of the prediction.
+ */
+double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100)
+{
+  RNN<MeanSquaredError<> > net(rho, true);
+  net.Add<LinearNoBias<> >(1, hiddenUnits);
+  net.Add<LSTM<> >(hiddenUnits, 1);
+
+  /*
+   * stepSize = 0.05, batchSize = 100, alpha = 0.9, epsilon = 1e-5,
+   * maxiterations = 4000, tolerance = 1e-5
+   *
+   */
+  RMSProp opt(0.05, 100, 0.9, 1e-08, 50000, 1e-5);
+
+  // Generate data
+  arma::cube data;
+  arma::cube labels;
+  GenerateNoisySinRNN(data, labels, rho, 1, 20000, 1.0, 200, 0.0, 45, 20);
+
+  // Break into training and test sets. Simply split along columns.
+  size_t     trainCols = data.n_cols * 0.8; // Take 20% out for testing.
+  size_t     testCols = data.n_cols - trainCols;
+  arma::cube testData =
+      data.subcube(0, data.n_cols - testCols, 0, data.n_rows - 1,
+                   data.n_cols - 1, data.n_slices - 1);
+  arma::cube testLabels =
+      labels.subcube(0, labels.n_cols - testCols, 0, labels.n_rows - 1,
+                     labels.n_cols - 1, labels.n_slices - 1);
+
+  for (int i = 0; i < numEpochs; ++i)
+  {
+    net.Train(data.subcube(0, 0, 0, data.n_rows - 1, trainCols - 1,
+                           data.n_slices - 1),
+              labels.subcube(0, 0, 0, labels.n_rows - 1, trainCols - 1,
+                             labels.n_slices - 1),
+              opt);
+  }
+  // Well now it should be trained. Do the test here.
+  arma::cube prediction;
+  net.Predict(testData, prediction);
+
+  // Take slice rho only.
+  arma::mat actualPred = prediction.slice(rho - 1);
+  // actualPred.print( "Prediction:" );
+  // testLabels.print( "Actual" );
+  double error =
+      arma::mean(arma::mean(arma::square(actualPred - testLabels.slice(0))));
+  return error;
+}
+
+/**
+ * Test RNN using multiple timestep input and single output.
+ */
+BOOST_AUTO_TEST_CASE(MultiTimestepTest)
+{
+  double err = RNNSineTest(112, 10, 10);
+  BOOST_REQUIRE_LE(err, 1e-02);
+}
+
 BOOST_AUTO_TEST_SUITE_END();

From a31d894bbf92b5629ee9c95d1375f7fcfbdfcbd3 Mon Sep 17 00:00:00 2001
From: Projyal Dev <projyal@gmail.com>
Date: Sun, 1 Apr 2018 07:26:59 +0530
Subject: [PATCH 19/67] Added tests for RNN for multi timestep input and single
 output.

---
 src/mlpack/tests/recurrent_network_test.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp
index 35bb7c95b3..36f76a9994 100644
--- a/src/mlpack/tests/recurrent_network_test.cpp
+++ b/src/mlpack/tests/recurrent_network_test.cpp
@@ -15,6 +15,7 @@
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/ann/rnn.hpp>
 #include <mlpack/core/data/binarize.hpp>
+#include <mlpack/core/optimizers/rmsprop/rmsprop.hpp>
 #include <cmath>
 #include <random>
 #include <chrono>

From 6715b75d9de15fb40e21d4c98401ee7c1a583aad Mon Sep 17 00:00:00 2001
From: Projyal Dev <projyal@gmail.com>
Date: Mon, 2 Apr 2018 18:02:11 +0530
Subject: [PATCH 20/67] Removed unwanted code that was saving the generated
 sine data.

---
 src/mlpack/tests/recurrent_network_test.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp
index 36f76a9994..36e72725a9 100644
--- a/src/mlpack/tests/recurrent_network_test.cpp
+++ b/src/mlpack/tests/recurrent_network_test.cpp
@@ -1146,7 +1146,6 @@ void GenerateNoisySinRNN(arma::cube& data, arma::cube& labels, size_t rho,
   arma::colvec y = x;
   if (normalize)
     y = arma::normalise(x);
-  y.save("rawsin.csv", arma::csv_ascii);
 
   // Now break this into columns of rho size slices.
   size_t n_columns = y.n_elem / rho;

From e2c729a232c820ac2cae51e90229529915061489 Mon Sep 17 00:00:00 2001
From: Projyal Dev <projyal@gmail.com>
Date: Tue, 3 Apr 2018 20:04:16 +0530
Subject: [PATCH 21/67] Review comments incorporated. Removed some unwanted
 code.

---
 src/mlpack/methods/ann/rnn_impl.hpp         |  2 --
 src/mlpack/tests/recurrent_network_test.cpp | 34 +++++++++------------
 2 files changed, 14 insertions(+), 22 deletions(-)

diff --git a/src/mlpack/methods/ann/rnn_impl.hpp b/src/mlpack/methods/ann/rnn_impl.hpp
index 211f87163a..5e5bef0db0 100644
--- a/src/mlpack/methods/ann/rnn_impl.hpp
+++ b/src/mlpack/methods/ann/rnn_impl.hpp
@@ -213,8 +213,6 @@ double RNN<OutputLayerType, InitializationRuleType, CustomLayers...>::Evaluate(
     {
       respSeq = seqNum;
     }
-    arma::mat respData(responses.slice(respSeq).colptr(begin),
-        responses.n_rows, batchSize, false, true);
 
     if (!deterministic)
     {
diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp
index 36e72725a9..25cd55c3c4 100644
--- a/src/mlpack/tests/recurrent_network_test.cpp
+++ b/src/mlpack/tests/recurrent_network_test.cpp
@@ -12,13 +12,11 @@
 #include <mlpack/core.hpp>
 
 #include <mlpack/core/optimizers/sgd/sgd.hpp>
+#include <mlpack/core/optimizers/rmsprop/rmsprop.hpp>
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/ann/rnn.hpp>
 #include <mlpack/core/data/binarize.hpp>
-#include <mlpack/core/optimizers/rmsprop/rmsprop.hpp>
-#include <cmath>
-#include <random>
-#include <chrono>
+#include <mlpack/core/math/random.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
@@ -28,6 +26,7 @@
 using namespace mlpack;
 using namespace mlpack::ann;
 using namespace mlpack::optimization;
+using namespace mlpack::math;
 
 BOOST_AUTO_TEST_SUITE(RecurrentNetworkTest);
 
@@ -1090,15 +1089,16 @@ BOOST_AUTO_TEST_CASE(CustomRecursiveReberGrammarTest)
 }
 
 /**
- * @brief Generates noisy sine wave.
+ * @brief Generates noisy sine wave and outputs the data and the labels that
+ * can be used directly for training and testing with RNN.
  *
  * @param data    The data points as output
  * @param labels  The expected values as output
  * @param rho     The size of the sequence of each data point
- * @param outputSteps How many output steps to consider
- * @param dataPoints  The number of input data points. The actual generated data
- * points may be more than this to adjust to the outputSteps. But at the minimum
- * these many data points will be generated.
+ * @param outputSteps How many output steps to consider for every rho inputs
+ * @param dataPoints  The number of generated data points. The actual generated
+ * data points may be more than this to adjust to the outputSteps.
+ * But at the minimum these many data points will be generated.
  * @param gain    The gain on the amplitude
  * @param freq    The frquency of the sine wave
  * @param phase   The phase shift if any
@@ -1128,19 +1128,13 @@ void GenerateNoisySinRNN(arma::cube& data, arma::cube& labels, size_t rho,
   arma::colvec    x(points);
   int             i = 0;
   double          interval = numCycles / freq / points;
-  std::mt19937_64 rng;
-  // initialize the random number generator with time-dependent seed
-  uint64_t timeSeed =
-      std::chrono::high_resolution_clock::now().time_since_epoch().count();
-  std::seed_seq ss{uint32_t(timeSeed & 0xffffffff), uint32_t(timeSeed >> 32)};
-  rng.seed(ss);
-  // initialize a uniform distribution between 0 and 1
-  std::uniform_real_distribution< double > unif(0, 1);
-  x.for_each([&i, gain, freq, phase, noisePercent, interval, &rng,
-              &unif](arma::colvec::elem_type& val) {
+
+  RandomSeed(20);
+  x.for_each([&i, gain, freq, phase, noisePercent, interval]
+             (arma::colvec::elem_type& val) {
     double t = interval * (i++);
     val = gain * ::sin(2 * M_PI * freq * t + phase) +
-          (noisePercent * gain / 100 * unif(rng));
+          (noisePercent * gain / 100 * Random(0.0, 0.1));
   });
 
   arma::colvec y = x;

From 591e57d93660c2599bdc7e910a44515833def2b4 Mon Sep 17 00:00:00 2001
From: aarushgupta <aarush98@gmail.com>
Date: Sun, 25 Feb 2018 21:58:18 +0530
Subject: [PATCH 22/67] Added flexibleReLU activation layer

---
 .../methods/ann/layer/flexible_relu.hpp       | 194 ++++++++++++++++++
 .../methods/ann/layer/flexible_relu_impl.hpp  |  61 ++++++
 2 files changed, 255 insertions(+)
 create mode 100644 src/mlpack/methods/ann/layer/flexible_relu.hpp
 create mode 100644 src/mlpack/methods/ann/layer/flexible_relu_impl.hpp

diff --git a/src/mlpack/methods/ann/layer/flexible_relu.hpp b/src/mlpack/methods/ann/layer/flexible_relu.hpp
new file mode 100644
index 0000000000..655a68c532
--- /dev/null
+++ b/src/mlpack/methods/ann/layer/flexible_relu.hpp
@@ -0,0 +1,194 @@
+/**
+ * @file flexible_relu.hpp
+ * @author Aarush Gupta
+ *
+ * Definition of FlexibleReLU layer as described by
+ * Suo Qiu, Xiangmin Xu and Bolun Cai in
+ * "FReLU: Flexible Rectified Linear Units for Improving Convolutional
+ *   Neural Networks", 2018
+ *
+ * For more information, read the following paper:
+ *
+ * @code
+ * @article{
+ *  author  = {Suo Qiu, Xiangmin Xu and Bolun Cai},
+ *  title   = {FReLU: Flexible Rectified Linear Units for Improving
+ *             Convolutional Neural Networks}
+ *  journal = {arxiv preprint},
+ *  year    = {2018}
+ * }
+ * @endcode
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_FLEXIBLERELU_HPP
+#define MLPACK_METHODS_ANN_LAYER_FLEXIBLERELU_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann /**Artificial Neural Network*/ {
+
+/**
+ *The FlexibleReLU activation function, defined by
+ *
+ * @f{eqnarray*}{
+ * f(x) &=& \max(0,x)+alpha \\
+ * f'(x) &=& \left\{
+ * 	 \begin(array){lr}
+ * 	   1 & : x > 0 \\
+ * 	   0 & : x \le 0
+ * 	 \end{array}
+ * \right
+ *@f}
+ *
+ *@tparam InputDataType Type of the input data ( arma::colvec, arma::mar,
+ *        arma::sp_mat or arma::cube)
+ *
+ *@tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *        arma::sp_mat or arma::cube)
+ *
+ */
+
+
+template <
+	     typename InputDataType = arma::mat,
+	     typename OutputDataType = arma::mat
+>
+class FlexibleReLU
+{
+public:
+  /**
+   *
+   * Create the FlexibleReLU object using the specified parameters.
+   * The non zero parameter can be adjusted by specifying the parameter
+   * alpha which controls the range of the relu function. ( Default alpha = 0)
+   * This parameter is trainable. 
+   *@param alpha Parameter for adjusting the range of the relu function.
+   *
+   */
+  FlexibleReLU(const double alpha = 0);
+
+  /**
+   * Ordinary feed forward pass of a neural network, evaluating the function
+   * f(x) by propagating the activity forward through f.
+   *
+   * @param input Input data used for evaluating the specified function.
+   * @param output Resulting output activation.
+   */
+  template<typename InputType, typename OutputType>
+  void Forward(const InputType&& input, OutputType&& output);
+
+  /**
+   * Ordinary feed backward pass of a neural network, calculating the function
+   * f(x) by propagating x backwards through f. Using the results from the feed
+   * forward pass.
+   *
+   * @param input The propagated input activation.
+   * @param gy The backpropagated error.
+   * @param g The calculated gradient.
+   */
+  template<typename DataType>
+  void Backward(const DataType&& input, DataType&& gy, DataType&& g);
+
+  //! Get the input parameter.
+  InputDataType const& InputParameter() const { return inputParameter; }
+  //!Modify the input parameter.
+  InputDataType& InputParameter() { return inputParameter; }
+
+  //!Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //!Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //!Modify the delta.
+  OutputDataType& Delta() { return delta;}
+  
+  //!Get the parameter controlling the range of the relu function.
+  double const& Alpha() const { return alpha; }
+  //!Modify the parameter controlling the range of the relu function.
+  double& Alpha() { return alpha; }
+
+  /**
+   * Serialize the layer.
+   */
+  template<typename Archive>
+  void serialize(Archive& ar, const unsigned int /* version*/);
+
+private:
+  /**
+   * Computes the FlexibleReLU function
+   *
+   * @param x Input data.
+   * @return f(x).
+   */
+ doubel Fn(const double x)
+ {
+ 	return (std::max(x,0) + alpha);
+ }
+
+ /**
+   * Computes the FlexibleReLU function using a dense matrix as input.
+   *
+   * @param x Input data.
+   * @param y The resulting output activation.
+   */
+ template<typename eT>
+ void Fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
+ {
+ 	y = arma::max(arma::zeros<arma::Mat<eT> >(x.n_rows, x.n_cols), x)+alpha;
+ }
+ 
+ /**
+  * Computes the first derivative of the LeakyReLU function.
+  *
+  * @param x Input data.
+  * @return f'(x)
+  */
+ double Deriv(const double x)
+ {
+   return x > 0;
+ }
+
+ /**
+   * Computes the first derivative of the FlexibleReLU function.
+   *
+   * @param y Input activations.
+   * @param The resulting dreivatives
+   */
+ template<typename InputType, typename, OutputType>
+ void Deriv(const InputParameter& x, OutputType& y)
+ {
+ 	y = x
+
+ 	for (size_t i = 0; i < x.n_elem; i++)
+	{
+	  y(i) = Deriv(x(i));
+	}
+ }
+
+ //! Locally-stored delta object.
+ OutputDataType delta;
+
+ //! Locally-stored input parameter object.
+ InputDataType inputParameter;
+
+ //! Locally-stored output parameter object.
+ OutputDataType outputParameter;
+
+ //! Parameter controlling the range of the rectifier function
+ double alpha;
+}; // class FlexibleReLU
+
+} // namespace ann
+} // namespace mlpack
+
+// Include implementation
+#include "flexible_relu_impl.hpp"
+
+#endif
\ No newline at end of file
diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
new file mode 100644
index 0000000000..69a30f2bc6
--- /dev/null
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -0,0 +1,61 @@
+/**
+ * @file flexible_relu_impl.hpp
+ * @author Aarush Gupta
+ *
+ * Implementation of FlexibleReLU layer as described by
+ * Suo Qiu, Xiangmin Xu and Bolun Cai in
+ * "FReLU: Flexible Rectified Linear Units for Improving Convolutional 
+ *  Neural Networks", 2018
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef MLPACK_METHODS_ANN_LAYER_FLEXIBLERELU_IMPL_HPP
+#define MLPACK_METHODS_ANN_LAYER_FLEXIBLERELU_IMPL_HPP
+
+#include "flexible_relu.hpp"
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+template<typename InputDataType, typename OutputDataType>
+FlexibleReLU<InputDataType, OutputDataType>::FlexibleReLU(
+	const double alpha) : aplha(alpha)
+{
+	//Nothing to do here.
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename InputType, typename OutputType>
+void FlexibleReLU<InputDataType, OutputDataType>::Forward(
+        const InputType&& input, OutputType&& output)
+{
+  Fn(input, output);
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename DataType>
+void FlexibleReLU<InputDataType, OutputDataType>::Backward(
+	const DataType&& input, DataType&& gy, DataType&& g)
+{
+	DataType derivative;
+	Deriv(input, derivative);
+	g = gy % derivative;
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename Archive>
+void FlexibleReLU<InputDataType, OutputDataType>::serialize(
+	Archive& ar,
+	const unsigned int /* version*/)
+{
+  ar & BOOST_SERIALIZATION_NVP(alpha);
+}
+
+} // napespace ann
+} // namespace mlpack
+
+#endif
\ No newline at end of file

From 2f179b6fca0db96bb28522840d36f93ec4ec0826 Mon Sep 17 00:00:00 2001
From: aarushgupta <aarush98@gmail.com>
Date: Mon, 26 Feb 2018 21:24:36 +0530
Subject: [PATCH 23/67] Added files to CMakeLists.txt

---
 src/mlpack/methods/ann/layer/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt
index 38dfa4377b..d4efc0ebf7 100644
--- a/src/mlpack/methods/ann/layer/CMakeLists.txt
+++ b/src/mlpack/methods/ann/layer/CMakeLists.txt
@@ -28,6 +28,8 @@ set(SOURCES
   elu_impl.hpp
   fast_lstm.hpp
   fast_lstm_impl.hpp
+  flexible_relu.hpp
+  flexible_relu_impl.hpp
   glimpse.hpp
   glimpse_impl.hpp
   gru.hpp

From 61c804ac2819f0319d60b32c9b0a0dfc0e20ec40 Mon Sep 17 00:00:00 2001
From: aarushgupta <aarush98@gmail.com>
Date: Wed, 28 Feb 2018 14:52:52 +0530
Subject: [PATCH 24/67] Added test for FReLU

---
 src/mlpack/tests/ann_layer_test.cpp | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp
index 448132efbf..65325b23ad 100644
--- a/src/mlpack/tests/ann_layer_test.cpp
+++ b/src/mlpack/tests/ann_layer_test.cpp
@@ -615,6 +615,25 @@ BOOST_AUTO_TEST_CASE(JacobianLeakyReLULayerTest)
   }
 }
 
+/**
+ * Jacobian FlexibleReLU module test.
+ */
+BOOST_AUTO_TEST_CASE(JacobianFlexibleReLULayerTest)
+{
+   for (size_t i = 0; i < 5; i++)
+   {
+   	const size_t inputElements = math::RandInt(2, 1000);
+
+   	arma::mat input;
+   	input.set_size(inputElements, 1);
+
+   	FlexibleReLU<> module;
+
+   	double error = JacobianTest(module, input);
+   	BOOST_REQUIRE_LE(error, 1e-5);
+   }
+}
+
 /**
  * Jacobian MultiplyConstant module test.
  */

From 9c92a0ed34918b4ddda6b8699dcc77b81eca29e6 Mon Sep 17 00:00:00 2001
From: aarushgupta <aarush98@gmail.com>
Date: Wed, 28 Feb 2018 15:10:16 +0530
Subject: [PATCH 25/67] Fixed some errors

---
 .../methods/ann/layer/flexible_relu.hpp       | 81 ++++++++++---------
 .../methods/ann/layer/flexible_relu_impl.hpp  | 16 ++--
 2 files changed, 49 insertions(+), 48 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu.hpp b/src/mlpack/methods/ann/layer/flexible_relu.hpp
index 655a68c532..f64b7e8057 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu.hpp
@@ -60,7 +60,7 @@ template <
 >
 class FlexibleReLU
 {
-public:
+ public:
   /**
    *
    * Create the FlexibleReLU object using the specified parameters.
@@ -120,69 +120,70 @@ public:
   template<typename Archive>
   void serialize(Archive& ar, const unsigned int /* version*/);
 
-private:
+ private:
   /**
    * Computes the FlexibleReLU function
    *
    * @param x Input data.
    * @return f(x).
    */
- doubel Fn(const double x)
- {
- 	return (std::max(x,0) + alpha);
- }
+  double Fn(const double x)
+  {
+ 	  return (std::max(x,0) + alpha);
+  }
 
- /**
+  /**
    * Computes the FlexibleReLU function using a dense matrix as input.
    *
    * @param x Input data.
    * @param y The resulting output activation.
    */
- template<typename eT>
- void Fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
- {
- 	y = arma::max(arma::zeros<arma::Mat<eT> >(x.n_rows, x.n_cols), x)+alpha;
- }
+  template<typename eT>
+  void Fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
+  {
+ 	  y = arma::max(arma::zeros<arma::Mat<eT> >(x.n_rows, x.n_cols), x)+alpha;
+  }
  
- /**
-  * Computes the first derivative of the LeakyReLU function.
-  *
-  * @param x Input data.
-  * @return f'(x)
-  */
- double Deriv(const double x)
- {
-   return x > 0;
- }
+  /**
+   * Computes the first derivative of the LeakyReLU function.
+   *
+   * @param x Input data.
+   * @return f'(x)
+   */
+  double Deriv(const double x)
+  {
+    return x > 0;
+  }
 
- /**
+  /**
    * Computes the first derivative of the FlexibleReLU function.
    *
    * @param y Input activations.
    * @param The resulting dreivatives
    */
- template<typename InputType, typename, OutputType>
- void Deriv(const InputParameter& x, OutputType& y)
- {
- 	y = x
+  
+  template<typename InputType, typename, OutputType>
+  void Deriv(const InputParameter& x, OutputType& y)
+  {
+ 	  y = x
 
- 	for (size_t i = 0; i < x.n_elem; i++)
-	{
-	  y(i) = Deriv(x(i));
-	}
- }
+ 	  for (size_t i = 0; i < x.n_elem; i++)
+	  {
+	    y(i) = Deriv(x(i));
+	  }
+  }
 
- //! Locally-stored delta object.
- OutputDataType delta;
+  //! Locally-stored delta object.
+  OutputDataType delta;
 
- //! Locally-stored input parameter object.
- InputDataType inputParameter;
+  //! Locally-stored input parameter object.
+  InputDataType inputParameter;
 
- //! Locally-stored output parameter object.
- OutputDataType outputParameter;
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
 
- //! Parameter controlling the range of the rectifier function
- double alpha;
+  //! Parameter controlling the range of the rectifier function
+  double alpha;
 }; // class FlexibleReLU
 
 } // namespace ann
diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index 69a30f2bc6..10810daf58 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -23,7 +23,7 @@ namespace ann /** Artificial Neural Network. */ {
 
 template<typename InputDataType, typename OutputDataType>
 FlexibleReLU<InputDataType, OutputDataType>::FlexibleReLU(
-	const double alpha) : aplha(alpha)
+    const double alpha) : aplha(alpha)
 {
 	//Nothing to do here.
 }
@@ -31,7 +31,7 @@ FlexibleReLU<InputDataType, OutputDataType>::FlexibleReLU(
 template<typename InputDataType, typename OutputDataType>
 template<typename InputType, typename OutputType>
 void FlexibleReLU<InputDataType, OutputDataType>::Forward(
-        const InputType&& input, OutputType&& output)
+    const InputType&& input, OutputType&& output)
 {
   Fn(input, output);
 }
@@ -39,18 +39,18 @@ void FlexibleReLU<InputDataType, OutputDataType>::Forward(
 template<typename InputDataType, typename OutputDataType>
 template<typename DataType>
 void FlexibleReLU<InputDataType, OutputDataType>::Backward(
-	const DataType&& input, DataType&& gy, DataType&& g)
+    const DataType&& input, DataType&& gy, DataType&& g)
 {
-	DataType derivative;
-	Deriv(input, derivative);
-	g = gy % derivative;
+    DataType derivative;
+    Deriv(input, derivative);
+    g = gy % derivative;
 }
 
 template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
 void FlexibleReLU<InputDataType, OutputDataType>::serialize(
-	Archive& ar,
-	const unsigned int /* version*/)
+    Archive& ar,
+    const unsigned int /* version*/)
 {
   ar & BOOST_SERIALIZATION_NVP(alpha);
 }

From 842214e8e3d48122f5d29adc1eb742cb5cb83249 Mon Sep 17 00:00:00 2001
From: aarushgupta <aarush98@gmail.com>
Date: Wed, 28 Feb 2018 15:22:40 +0530
Subject: [PATCH 26/67] Fixed style errors

---
 src/mlpack/methods/ann/layer/flexible_relu.hpp | 18 +++++++++---------
 .../methods/ann/layer/flexible_relu_impl.hpp   |  8 ++++----
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu.hpp b/src/mlpack/methods/ann/layer/flexible_relu.hpp
index f64b7e8057..4d7794e459 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu.hpp
@@ -55,8 +55,8 @@ namespace ann /**Artificial Neural Network*/ {
 
 
 template <
-	     typename InputDataType = arma::mat,
-	     typename OutputDataType = arma::mat
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
 >
 class FlexibleReLU
 {
@@ -129,7 +129,7 @@ class FlexibleReLU
    */
   double Fn(const double x)
   {
- 	  return (std::max(x,0) + alpha);
+    return (std::max(x,0) + alpha);
   }
 
   /**
@@ -141,7 +141,7 @@ class FlexibleReLU
   template<typename eT>
   void Fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
   {
- 	  y = arma::max(arma::zeros<arma::Mat<eT> >(x.n_rows, x.n_cols), x)+alpha;
+    y = arma::max(arma::zeros<arma::Mat<eT> >(x.n_rows, x.n_cols), x)+alpha;
   }
  
   /**
@@ -165,12 +165,12 @@ class FlexibleReLU
   template<typename InputType, typename, OutputType>
   void Deriv(const InputParameter& x, OutputType& y)
   {
- 	  y = x
+    y = x
 
- 	  for (size_t i = 0; i < x.n_elem; i++)
-	  {
-	    y(i) = Deriv(x(i));
-	  }
+    for (size_t i = 0; i < x.n_elem; i++)
+    {
+      y(i) = Deriv(x(i));
+    }
   }
 
   //! Locally-stored delta object.
diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index 10810daf58..fb8acd59b0 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -25,7 +25,7 @@ template<typename InputDataType, typename OutputDataType>
 FlexibleReLU<InputDataType, OutputDataType>::FlexibleReLU(
     const double alpha) : aplha(alpha)
 {
-	//Nothing to do here.
+  //Nothing to do here.
 }
 
 template<typename InputDataType, typename OutputDataType>
@@ -41,9 +41,9 @@ template<typename DataType>
 void FlexibleReLU<InputDataType, OutputDataType>::Backward(
     const DataType&& input, DataType&& gy, DataType&& g)
 {
-    DataType derivative;
-    Deriv(input, derivative);
-    g = gy % derivative;
+  DataType derivative;
+  Deriv(input, derivative);
+  g = gy % derivative;
 }
 
 template<typename InputDataType, typename OutputDataType>

From 4571a7dd09e698456de079f0590b1e247428315f Mon Sep 17 00:00:00 2001
From: aarushgupta <aarush98@gmail.com>
Date: Wed, 28 Feb 2018 16:44:23 +0530
Subject: [PATCH 27/67] Fixed build errors

---
 src/mlpack/methods/ann/layer/flexible_relu.hpp      | 8 ++++----
 src/mlpack/methods/ann/layer/flexible_relu_impl.hpp | 2 +-
 src/mlpack/methods/ann/layer/layer_types.hpp        | 2 ++
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu.hpp b/src/mlpack/methods/ann/layer/flexible_relu.hpp
index 4d7794e459..a286811e88 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu.hpp
@@ -129,7 +129,7 @@ class FlexibleReLU
    */
   double Fn(const double x)
   {
-    return (std::max(x,0) + alpha);
+    return (std::max(x, 0 * x) + alpha);
   }
 
   /**
@@ -162,10 +162,10 @@ class FlexibleReLU
    * @param The resulting dreivatives
    */
   
-  template<typename InputType, typename, OutputType>
-  void Deriv(const InputParameter& x, OutputType& y)
+  template<typename InputType, typename OutputType>
+  void Deriv(const InputType& x, OutputType& y)
   {
-    y = x
+    y = x;
 
     for (size_t i = 0; i < x.n_elem; i++)
     {
diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index fb8acd59b0..1c9471d58c 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -23,7 +23,7 @@ namespace ann /** Artificial Neural Network. */ {
 
 template<typename InputDataType, typename OutputDataType>
 FlexibleReLU<InputDataType, OutputDataType>::FlexibleReLU(
-    const double alpha) : aplha(alpha)
+    const double alpha) : alpha(alpha)
 {
   //Nothing to do here.
 }
diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp
index 9d609effdd..1587efbfa8 100644
--- a/src/mlpack/methods/ann/layer/layer_types.hpp
+++ b/src/mlpack/methods/ann/layer/layer_types.hpp
@@ -26,6 +26,7 @@
 #include <mlpack/methods/ann/layer/hard_tanh.hpp>
 #include <mlpack/methods/ann/layer/join.hpp>
 #include <mlpack/methods/ann/layer/leaky_relu.hpp>
+#include <mlpack/methods/ann/layer/flexible_relu.hpp>
 #include <mlpack/methods/ann/layer/log_softmax.hpp>
 #include <mlpack/methods/ann/layer/lookup.hpp>
 #include <mlpack/methods/ann/layer/mean_squared_error.hpp>
@@ -124,6 +125,7 @@ using LayerTypes = boost::variant<
     DropConnect<arma::mat, arma::mat>*,
     Dropout<arma::mat, arma::mat>*,
     ELU<arma::mat, arma::mat>*,
+    FlexibleReLU<arma::mat, arma::mat>*,
     Glimpse<arma::mat, arma::mat>*,
     HardTanH<arma::mat, arma::mat>*,
     Join<arma::mat, arma::mat>*,

From 49efbbd94c753de86965c27fc8ad9e08d3ea8684 Mon Sep 17 00:00:00 2001
From: aarushgupta <aarush98@gmail.com>
Date: Wed, 28 Feb 2018 17:57:53 +0530
Subject: [PATCH 28/67] Modified ann_layer_test.cpp

---
 .../methods/ann/layer/flexible_relu.hpp       | 22 +++++++++----------
 .../methods/ann/layer/flexible_relu_impl.hpp  |  6 ++---
 src/mlpack/tests/ann_layer_test.cpp           | 18 +++++++--------
 3 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu.hpp b/src/mlpack/methods/ann/layer/flexible_relu.hpp
index a286811e88..64060e083a 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu.hpp
@@ -96,22 +96,22 @@ class FlexibleReLU
 
   //! Get the input parameter.
   InputDataType const& InputParameter() const { return inputParameter; }
-  //!Modify the input parameter.
+  //! Modify the input parameter.
   InputDataType& InputParameter() { return inputParameter; }
 
-  //!Get the output parameter.
+  //! Get the output parameter.
   OutputDataType const& OutputParameter() const { return outputParameter; }
-  //Modify the output parameter.
+  //! Modify the output parameter.
   OutputDataType& OutputParameter() { return outputParameter; }
 
-  //!Get the delta.
+  //! Get the delta.
   OutputDataType const& Delta() const { return delta; }
-  //!Modify the delta.
+  //! Modify the delta.
   OutputDataType& Delta() { return delta;}
-  
-  //!Get the parameter controlling the range of the relu function.
+
+  //! Get the parameter controlling the range of the relu function.
   double const& Alpha() const { return alpha; }
-  //!Modify the parameter controlling the range of the relu function.
+  //! Modify the parameter controlling the range of the relu function.
   double& Alpha() { return alpha; }
 
   /**
@@ -143,7 +143,7 @@ class FlexibleReLU
   {
     y = arma::max(arma::zeros<arma::Mat<eT> >(x.n_rows, x.n_cols), x)+alpha;
   }
- 
+
   /**
    * Computes the first derivative of the LeakyReLU function.
    *
@@ -161,7 +161,7 @@ class FlexibleReLU
    * @param y Input activations.
    * @param The resulting dreivatives
    */
-  
+
   template<typename InputType, typename OutputType>
   void Deriv(const InputType& x, OutputType& y)
   {
@@ -192,4 +192,4 @@ class FlexibleReLU
 // Include implementation
 #include "flexible_relu_impl.hpp"
 
-#endif
\ No newline at end of file
+#endif
diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index 1c9471d58c..78200f44b9 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -25,7 +25,7 @@ template<typename InputDataType, typename OutputDataType>
 FlexibleReLU<InputDataType, OutputDataType>::FlexibleReLU(
     const double alpha) : alpha(alpha)
 {
-  //Nothing to do here.
+  // Nothing to do here.
 }
 
 template<typename InputDataType, typename OutputDataType>
@@ -55,7 +55,7 @@ void FlexibleReLU<InputDataType, OutputDataType>::serialize(
   ar & BOOST_SERIALIZATION_NVP(alpha);
 }
 
-} // napespace ann
+} // namespace ann
 } // namespace mlpack
 
-#endif
\ No newline at end of file
+#endif
diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp
index 65325b23ad..6dd706a20c 100644
--- a/src/mlpack/tests/ann_layer_test.cpp
+++ b/src/mlpack/tests/ann_layer_test.cpp
@@ -620,18 +620,18 @@ BOOST_AUTO_TEST_CASE(JacobianLeakyReLULayerTest)
  */
 BOOST_AUTO_TEST_CASE(JacobianFlexibleReLULayerTest)
 {
-   for (size_t i = 0; i < 5; i++)
-   {
-   	const size_t inputElements = math::RandInt(2, 1000);
+  for (size_t i = 0; i < 5; i++)
+  {
+    const size_t inputElements = math::RandInt(2, 1000);
 
-   	arma::mat input;
-   	input.set_size(inputElements, 1);
+    arma::mat input;
+    input.set_size(inputElements, 1);
 
-   	FlexibleReLU<> module;
+    FlexibleReLU<> module;
 
-   	double error = JacobianTest(module, input);
-   	BOOST_REQUIRE_LE(error, 1e-5);
-   }
+    double error = JacobianTest(module, input);
+    BOOST_REQUIRE_LE(error, 1e-5);
+  }
 }
 
 /**

From 0f38dc173077881d0352aabab95008fb7893edee Mon Sep 17 00:00:00 2001
From: aarushgupta <aarush98@gmail.com>
Date: Wed, 28 Feb 2018 23:34:30 +0530
Subject: [PATCH 29/67] Fixed typecasting error

---
 src/mlpack/methods/ann/layer/flexible_relu.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu.hpp b/src/mlpack/methods/ann/layer/flexible_relu.hpp
index 64060e083a..75bf8a4619 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu.hpp
@@ -152,7 +152,7 @@ class FlexibleReLU
    */
   double Deriv(const double x)
   {
-    return x > 0;
+    return x > 0? 1 : 0;
   }
 
   /**

From 0c4b121a46fb5cb0a682217fa39e30e5b022bd2b Mon Sep 17 00:00:00 2001
From: aarushgupta <aarush98@gmail.com>
Date: Thu, 1 Mar 2018 14:25:15 +0530
Subject: [PATCH 30/67] Modified comments

---
 src/mlpack/methods/ann/layer/flexible_relu_impl.hpp | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index 78200f44b9..b70e0fc820 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -7,6 +7,18 @@
  * "FReLU: Flexible Rectified Linear Units for Improving Convolutional 
  *  Neural Networks", 2018
  *
+ * For more information, read the following paper:
+ *
+ * @code
+ * @article{
+ *  author  = {Suo Qiu, Xiangmin Xu and Bolun Cai},
+ *  title   = {FReLU: Flexible Rectified Linear Units for Improving
+ *             Convolutional Neural Networks}
+ *  journal = {arxiv preprint},
+ *  year    = {2018}
+ * }
+ * @endcode
+ *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
  * 3-clause BSD license along with mlpack.  If not, see

From 0c5dc0c0af11979af36fc309079009fe3b13eea8 Mon Sep 17 00:00:00 2001
From: manthan-r-sheth <manthanrsheth96@gmail.com>
Date: Fri, 30 Mar 2018 15:10:31 +0530
Subject: [PATCH 31/67] Simplify code to reduce function call overhead

---
 .../methods/ann/layer/flexible_relu.hpp       | 51 -------------------
 .../methods/ann/layer/flexible_relu_impl.hpp  | 13 ++++-
 src/mlpack/methods/ann/layer/leaky_relu.hpp   |  4 +-
 3 files changed, 13 insertions(+), 55 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu.hpp b/src/mlpack/methods/ann/layer/flexible_relu.hpp
index 75bf8a4619..2cf3a0d9e4 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu.hpp
@@ -121,57 +121,6 @@ class FlexibleReLU
   void serialize(Archive& ar, const unsigned int /* version*/);
 
  private:
-  /**
-   * Computes the FlexibleReLU function
-   *
-   * @param x Input data.
-   * @return f(x).
-   */
-  double Fn(const double x)
-  {
-    return (std::max(x, 0 * x) + alpha);
-  }
-
-  /**
-   * Computes the FlexibleReLU function using a dense matrix as input.
-   *
-   * @param x Input data.
-   * @param y The resulting output activation.
-   */
-  template<typename eT>
-  void Fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
-  {
-    y = arma::max(arma::zeros<arma::Mat<eT> >(x.n_rows, x.n_cols), x)+alpha;
-  }
-
-  /**
-   * Computes the first derivative of the LeakyReLU function.
-   *
-   * @param x Input data.
-   * @return f'(x)
-   */
-  double Deriv(const double x)
-  {
-    return x > 0? 1 : 0;
-  }
-
-  /**
-   * Computes the first derivative of the FlexibleReLU function.
-   *
-   * @param y Input activations.
-   * @param The resulting dreivatives
-   */
-
-  template<typename InputType, typename OutputType>
-  void Deriv(const InputType& x, OutputType& y)
-  {
-    y = x;
-
-    for (size_t i = 0; i < x.n_elem; i++)
-    {
-      y(i) = Deriv(x(i));
-    }
-  }
 
   //! Locally-stored delta object.
   OutputDataType delta;
diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index b70e0fc820..c3effff128 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -45,7 +45,8 @@ template<typename InputType, typename OutputType>
 void FlexibleReLU<InputDataType, OutputDataType>::Forward(
     const InputType&& input, OutputType&& output)
 {
-  Fn(input, output);
+  output = arma::max(arma::zeros<InputType>(input.n_rows, input.n_cols), input)
+    + alpha;
 }
 
 template<typename InputDataType, typename OutputDataType>
@@ -54,7 +55,15 @@ void FlexibleReLU<InputDataType, OutputDataType>::Backward(
     const DataType&& input, DataType&& gy, DataType&& g)
 {
   DataType derivative;
-  Deriv(input, derivative);
+
+  //! Compute the first derivative of FlexibleReLU function.
+  derivative = input;
+
+  for (size_t i = 0; i < input.n_elem; i++)
+  {
+    derivative(i) = input(i) > 0? 1 : 0;
+  }
+
   g = gy % derivative;
 }
 
diff --git a/src/mlpack/methods/ann/layer/leaky_relu.hpp b/src/mlpack/methods/ann/layer/leaky_relu.hpp
index 88d2b662d7..dfa7af8e92 100644
--- a/src/mlpack/methods/ann/layer/leaky_relu.hpp
+++ b/src/mlpack/methods/ann/layer/leaky_relu.hpp
@@ -103,7 +103,7 @@ class LeakyReLU
 
  private:
   /**
-   * Computes the LeakReLU function
+   * Computes the LeakyReLU function
    *
    * @param x Input data.
    * @return f(x).
@@ -114,7 +114,7 @@ class LeakyReLU
   }
 
   /**
-   * Computes the Leaky ReLU function using a dense matrix as input.
+   * Computes the LeakyReLU function using a dense matrix as input.
    *
    * @param x Input data.
    * @param y The resulting output activation.

From d32616b06f387d1d2790ac1985796cf881ed31d8 Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Fri, 30 Mar 2018 17:15:48 +0530
Subject: [PATCH 32/67] Add URL and author name

---
 src/mlpack/methods/ann/layer/flexible_relu.hpp      | 2 ++
 src/mlpack/methods/ann/layer/flexible_relu_impl.hpp | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu.hpp b/src/mlpack/methods/ann/layer/flexible_relu.hpp
index 2cf3a0d9e4..9c4ba3819d 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu.hpp
@@ -1,6 +1,7 @@
 /**
  * @file flexible_relu.hpp
  * @author Aarush Gupta
+ * @author Manthan-R-Sheth
  *
  * Definition of FlexibleReLU layer as described by
  * Suo Qiu, Xiangmin Xu and Bolun Cai in
@@ -15,6 +16,7 @@
  *  title   = {FReLU: Flexible Rectified Linear Units for Improving
  *             Convolutional Neural Networks}
  *  journal = {arxiv preprint},
+ *  URL     = {https://arxiv.org/pdf/1706.08098.pdf},
  *  year    = {2018}
  * }
  * @endcode
diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index c3effff128..9e3ac8eb5e 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -1,6 +1,7 @@
 /**
  * @file flexible_relu_impl.hpp
  * @author Aarush Gupta
+ * @author Manthan-R-Sheth
  *
  * Implementation of FlexibleReLU layer as described by
  * Suo Qiu, Xiangmin Xu and Bolun Cai in
@@ -15,6 +16,7 @@
  *  title   = {FReLU: Flexible Rectified Linear Units for Improving
  *             Convolutional Neural Networks}
  *  journal = {arxiv preprint},
+ *  URL     = {https://arxiv.org/pdf/1706.08098.pdf},
  *  year    = {2018}
  * }
  * @endcode

From 7e926996c5bbf86f6c923bd71e9640433f5856cf Mon Sep 17 00:00:00 2001
From: manthan-r-sheth <manthanrsheth96@gmail.com>
Date: Mon, 2 Apr 2018 01:29:58 +0530
Subject: [PATCH 33/67] fix citations

---
 .../methods/ann/layer/flexible_relu.hpp       | 31 +++++++++----------
 .../methods/ann/layer/flexible_relu_impl.hpp  | 15 +--------
 2 files changed, 15 insertions(+), 31 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu.hpp b/src/mlpack/methods/ann/layer/flexible_relu.hpp
index 9c4ba3819d..6f40e4ce65 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu.hpp
@@ -8,19 +8,6 @@
  * "FReLU: Flexible Rectified Linear Units for Improving Convolutional
  *   Neural Networks", 2018
  *
- * For more information, read the following paper:
- *
- * @code
- * @article{
- *  author  = {Suo Qiu, Xiangmin Xu and Bolun Cai},
- *  title   = {FReLU: Flexible Rectified Linear Units for Improving
- *             Convolutional Neural Networks}
- *  journal = {arxiv preprint},
- *  URL     = {https://arxiv.org/pdf/1706.08098.pdf},
- *  year    = {2018}
- * }
- * @endcode
- *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
  * 3-clause BSD license along with mlpack.  If not, see
@@ -47,15 +34,25 @@ namespace ann /**Artificial Neural Network*/ {
  * \right
  *@f}
  *
- *@tparam InputDataType Type of the input data ( arma::colvec, arma::mar,
- *        arma::sp_mat or arma::cube)
+ * For more information, read the following paper:
  *
+ * @code
+ * @article{Qiu2018,
+ *  author  = {Suo Qiu, Xiangmin Xu and Bolun Cai},
+ *  title   = {FReLU: Flexible Rectified Linear Units for Improving
+ *             Convolutional Neural Networks}
+ *  journal = {arxiv preprint},
+ *  URL     = {https://arxiv.org/abs/1706.08098},
+ *  year    = {2018}
+ * }
+ * @endcode
+ *
+ *@tparam InputDataType Type of the input data (arma::colvec, arma::mar,
+ *        arma::sp_mat or arma::cube)
  *@tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
  *        arma::sp_mat or arma::cube)
  *
  */
-
-
 template <
     typename InputDataType = arma::mat,
     typename OutputDataType = arma::mat
diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index 9e3ac8eb5e..81803e5987 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -7,20 +7,7 @@
  * Suo Qiu, Xiangmin Xu and Bolun Cai in
  * "FReLU: Flexible Rectified Linear Units for Improving Convolutional 
  *  Neural Networks", 2018
- *
- * For more information, read the following paper:
- *
- * @code
- * @article{
- *  author  = {Suo Qiu, Xiangmin Xu and Bolun Cai},
- *  title   = {FReLU: Flexible Rectified Linear Units for Improving
- *             Convolutional Neural Networks}
- *  journal = {arxiv preprint},
- *  URL     = {https://arxiv.org/pdf/1706.08098.pdf},
- *  year    = {2018}
- * }
- * @endcode
- *
+ *  
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
  * 3-clause BSD license along with mlpack.  If not, see

From cf47d4b2d5741e41a91e98daa79526288636a40f Mon Sep 17 00:00:00 2001
From: manthan-r-sheth <manthanrsheth96@gmail.com>
Date: Mon, 2 Apr 2018 01:36:40 +0530
Subject: [PATCH 34/67] fix style errors

---
 src/mlpack/methods/ann/layer/flexible_relu.hpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu.hpp b/src/mlpack/methods/ann/layer/flexible_relu.hpp
index 6f40e4ce65..b0ac8fdd89 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu.hpp
@@ -120,7 +120,6 @@ class FlexibleReLU
   void serialize(Archive& ar, const unsigned int /* version*/);
 
  private:
-
   //! Locally-stored delta object.
   OutputDataType delta;
 

From b72d5d2a5d26a41ea03b443208c2725ae71872ba Mon Sep 17 00:00:00 2001
From: manthan-r-sheth <manthanrsheth96@gmail.com>
Date: Tue, 3 Apr 2018 10:14:52 +0530
Subject: [PATCH 35/67] Add gradient() and lambdas

---
 .../methods/ann/layer/flexible_relu.hpp       | 40 ++++++++++++++--
 .../methods/ann/layer/flexible_relu_impl.hpp  | 47 ++++++++++++++-----
 2 files changed, 73 insertions(+), 14 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu.hpp b/src/mlpack/methods/ann/layer/flexible_relu.hpp
index b0ac8fdd89..97fcaa9cbb 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu.hpp
@@ -65,11 +65,17 @@ class FlexibleReLU
    * Create the FlexibleReLU object using the specified parameters.
    * The non zero parameter can be adjusted by specifying the parameter
    * alpha which controls the range of the relu function. ( Default alpha = 0)
-   * This parameter is trainable. 
+   * This parameter is trainable.
+   *
    *@param alpha Parameter for adjusting the range of the relu function.
    *
    */
-  FlexibleReLU(const double alpha = 0);
+  FlexibleReLU(const double userAlpha = 0);
+
+  /*
+   * Reset the layer parameter.
+   */
+  void Reset();
 
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function
@@ -93,6 +99,23 @@ class FlexibleReLU
   template<typename DataType>
   void Backward(const DataType&& input, DataType&& gy, DataType&& g);
 
+  /**
+   * Calculate the gradient using the output delta and the input activation.
+   *
+   * @param input The input parameter used for calculating the gradient.
+   * @param error The calculated error.
+   * @param gradient The calculated gradient.
+   */
+  template<typename eT>
+  void Gradient(const arma::Mat<eT>&& input,
+                arma::Mat<eT>&& error,
+                arma::Mat<eT>&& gradient);
+
+  //! Get the parameters.
+  OutputDataType const& Parameters() const { return alpha; }
+  //! Modify the parameters.
+  OutputDataType& Parameters() { return alpha; }
+
   //! Get the input parameter.
   InputDataType const& InputParameter() const { return inputParameter; }
   //! Modify the input parameter.
@@ -108,6 +131,11 @@ class FlexibleReLU
   //! Modify the delta.
   OutputDataType& Delta() { return delta;}
 
+  //! Get the gradient.
+  OutputDataType const& Gradient() const { return gradient; }
+  //! Modify the gradient.
+  OutputDataType& Gradient() { return gradient; }
+
   //! Get the parameter controlling the range of the relu function.
   double const& Alpha() const { return alpha; }
   //! Modify the parameter controlling the range of the relu function.
@@ -129,8 +157,14 @@ class FlexibleReLU
   //! Locally-stored output parameter object.
   OutputDataType outputParameter;
 
+  //! Leakyness Parameter object.
+  OutputDataType alpha;
+
+  //! Locally-stored gradient object.
+  OutputDataType gradient;
+
   //! Parameter controlling the range of the rectifier function
-  double alpha;
+  double userAlpha;
 }; // class FlexibleReLU
 
 } // namespace ann
diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index 81803e5987..c2b52d61cb 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -18,15 +18,24 @@
 #define MLPACK_METHODS_ANN_LAYER_FLEXIBLERELU_IMPL_HPP
 
 #include "flexible_relu.hpp"
+#include<algorithm>
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
 template<typename InputDataType, typename OutputDataType>
 FlexibleReLU<InputDataType, OutputDataType>::FlexibleReLU(
-    const double alpha) : alpha(alpha)
+    const double userAlpha) : userAlpha(userAlpha)
 {
-  // Nothing to do here.
+  alpha.set_size(1, 1);
+  alpha(0) = userAlpha;
+}
+
+template<typename InputDataType, typename OutputDataType>
+void FlexibleReLU<InputDataType, OutputDataType>::Reset()
+{
+  //! Set value of alpha to the one given by user.
+  alpha(0) = userAlpha;
 }
 
 template<typename InputDataType, typename OutputDataType>
@@ -34,8 +43,11 @@ template<typename InputType, typename OutputType>
 void FlexibleReLU<InputDataType, OutputDataType>::Forward(
     const InputType&& input, OutputType&& output)
 {
-  output = arma::max(arma::zeros<InputType>(input.n_rows, input.n_cols), input)
-    + alpha;
+  int i = -1;
+  output = arma::zeros<InputType>(input.n_rows, input.n_cols);
+  output.transform( [input, &i](double val) { ++i;
+      return (std::max(input(i), 0.0) + alpha); } );
+
 }
 
 template<typename InputDataType, typename OutputDataType>
@@ -44,18 +56,31 @@ void FlexibleReLU<InputDataType, OutputDataType>::Backward(
     const DataType&& input, DataType&& gy, DataType&& g)
 {
   DataType derivative;
-
   //! Compute the first derivative of FlexibleReLU function.
-  derivative = input;
-
-  for (size_t i = 0; i < input.n_elem; i++)
-  {
-    derivative(i) = input(i) > 0? 1 : 0;
-  }
+  derivative.set_size(input.n_rows, input.n_cols);
+  int i = -1;
+  derivative.transform( [input, &i](double val) { ++i;
+    return (input(i) > 0? 1 : 0); } );
 
   g = gy % derivative;
 }
 
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void FlexibleReLU<InputDataType, OutputDataType>::Gradient(
+  const arma::Mat<eT>&& input, arma::Mat<eT>&& error,
+  arma::Mat<eT>&& gradient)
+{
+  if (gradient.n_elem == 0)
+  {
+    gradient = arma::zeros<arma::Mat<eT>>(1, 1);
+  }
+
+  arma::mat zeros = arma::zeros<arma::Mat<eT>>(input.n_rows, input.n_cols);
+  gradient(0) = arma::accu(error % arma::min(zeros, input)) / input.n_cols;
+}
+
+
 template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
 void FlexibleReLU<InputDataType, OutputDataType>::serialize(

From 4f724a59a2d64f2c2ce4690dc8d951460ae4b6f4 Mon Sep 17 00:00:00 2001
From: manthan-r-sheth <manthanrsheth96@gmail.com>
Date: Tue, 3 Apr 2018 10:18:37 +0530
Subject: [PATCH 36/67] Fix style errors

---
 src/mlpack/methods/ann/layer/flexible_relu_impl.hpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index c2b52d61cb..25ce8faf83 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -45,9 +45,8 @@ void FlexibleReLU<InputDataType, OutputDataType>::Forward(
 {
   int i = -1;
   output = arma::zeros<InputType>(input.n_rows, input.n_cols);
-  output.transform( [input, &i](double val) { ++i;
+  output.transform([input, &i](double val) { ++i;
       return (std::max(input(i), 0.0) + alpha); } );
-
 }
 
 template<typename InputDataType, typename OutputDataType>
@@ -59,7 +58,7 @@ void FlexibleReLU<InputDataType, OutputDataType>::Backward(
   //! Compute the first derivative of FlexibleReLU function.
   derivative.set_size(input.n_rows, input.n_cols);
   int i = -1;
-  derivative.transform( [input, &i](double val) { ++i;
+  derivative.transform([input, &i](double val) { ++i;
     return (input(i) > 0? 1 : 0); } );
 
   g = gy % derivative;

From 051e29d0e8f8c8cd359e2830c286a24269ade1db Mon Sep 17 00:00:00 2001
From: manthan-r-sheth <manthanrsheth96@gmail.com>
Date: Tue, 3 Apr 2018 10:59:37 +0530
Subject: [PATCH 37/67] update lambda

---
 src/mlpack/methods/ann/layer/flexible_relu_impl.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index 25ce8faf83..45f477e9e1 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -45,8 +45,8 @@ void FlexibleReLU<InputDataType, OutputDataType>::Forward(
 {
   int i = -1;
   output = arma::zeros<InputType>(input.n_rows, input.n_cols);
-  output.transform([input, &i](double val) { ++i;
-      return (std::max(input(i), 0.0) + alpha); } );
+  output.transform([input, &i, alpha](double val) { ++i;
+      return (std::max(input(i), 0.0) + alpha(0)); } );
 }
 
 template<typename InputDataType, typename OutputDataType>

From 5ed72478c42a352671bcd09e2b91f3a9dcb8f2c9 Mon Sep 17 00:00:00 2001
From: manthan-r-sheth <manthanrsheth96@gmail.com>
Date: Tue, 3 Apr 2018 12:04:36 +0530
Subject: [PATCH 38/67] include this in lambda

---
 src/mlpack/methods/ann/layer/flexible_relu.hpp      | 2 +-
 src/mlpack/methods/ann/layer/flexible_relu_impl.hpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu.hpp b/src/mlpack/methods/ann/layer/flexible_relu.hpp
index 97fcaa9cbb..7b08fb3a8f 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu.hpp
@@ -157,7 +157,7 @@ class FlexibleReLU
   //! Locally-stored output parameter object.
   OutputDataType outputParameter;
 
-  //! Leakyness Parameter object.
+  //! Parameter object.
   OutputDataType alpha;
 
   //! Locally-stored gradient object.
diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index 45f477e9e1..70e03de95f 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -45,7 +45,7 @@ void FlexibleReLU<InputDataType, OutputDataType>::Forward(
 {
   int i = -1;
   output = arma::zeros<InputType>(input.n_rows, input.n_cols);
-  output.transform([input, &i, alpha](double val) { ++i;
+  output.transform([input, &i, this](double val) { ++i;
       return (std::max(input(i), 0.0) + alpha(0)); } );
 }
 

From b3451ab01716dcd57edcec034b7e6bc7f39d6083 Mon Sep 17 00:00:00 2001
From: manthan-r-sheth <manthanrsheth96@gmail.com>
Date: Wed, 4 Apr 2018 01:34:19 +0530
Subject: [PATCH 39/67] add checkgradient and update passes

---
 .../methods/ann/layer/flexible_relu_impl.hpp  | 17 +++-----
 src/mlpack/tests/ann_layer_test.cpp           | 42 +++++++++++++++++++
 2 files changed, 47 insertions(+), 12 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index 70e03de95f..2fa1c5dca6 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -43,10 +43,7 @@ template<typename InputType, typename OutputType>
 void FlexibleReLU<InputDataType, OutputDataType>::Forward(
     const InputType&& input, OutputType&& output)
 {
-  int i = -1;
-  output = arma::zeros<InputType>(input.n_rows, input.n_cols);
-  output.transform([input, &i, this](double val) { ++i;
-      return (std::max(input(i), 0.0) + alpha(0)); } );
+  output = arma::clamp(input, 0.0, DBL_MAX) + alpha(0);
 }
 
 template<typename InputDataType, typename OutputDataType>
@@ -56,11 +53,8 @@ void FlexibleReLU<InputDataType, OutputDataType>::Backward(
 {
   DataType derivative;
   //! Compute the first derivative of FlexibleReLU function.
-  derivative.set_size(input.n_rows, input.n_cols);
-  int i = -1;
-  derivative.transform([input, &i](double val) { ++i;
-    return (input(i) > 0? 1 : 0); } );
-
+  derivative = arma::sign(input);
+  derivative.elem(arma::find(derivative < 0.0)) += 1;
   g = gy % derivative;
 }
 
@@ -74,9 +68,8 @@ void FlexibleReLU<InputDataType, OutputDataType>::Gradient(
   {
     gradient = arma::zeros<arma::Mat<eT>>(1, 1);
   }
-
-  arma::mat zeros = arma::zeros<arma::Mat<eT>>(input.n_rows, input.n_cols);
-  gradient(0) = arma::accu(error % arma::min(zeros, input)) / input.n_cols;
+  gradient(0) = arma::accu(error % arma::clamp(input, -DBL_MAX, 0.0))
+      / input.n_cols;
 }
 
 
diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp
index 6dd706a20c..57b71cb015 100644
--- a/src/mlpack/tests/ann_layer_test.cpp
+++ b/src/mlpack/tests/ann_layer_test.cpp
@@ -634,6 +634,48 @@ BOOST_AUTO_TEST_CASE(JacobianFlexibleReLULayerTest)
   }
 }
 
+/**
+ * Flexible ReLU layer numerically gradient test.
+ */
+BOOST_AUTO_TEST_CASE(GradientFlexibleReLULayerTest)
+{
+  // Add function gradient instantiation.
+  struct GradientFunction
+  {
+    GradientFunction()
+    {
+      input = arma::randn(10, 1);
+      target = arma::mat("1");
+
+      model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>(
+        input, target);
+      model->Add<Linear<> >(10, 2);
+      model->Add<FlexibleReLU<> >(0.05);
+      model->Add<LogSoftMax<> >();
+    }
+
+    ~GradientFunction()
+    {
+      delete model;
+    }
+
+    double Gradient(arma::mat& gradient) const
+    {
+      arma::mat output;
+      double error = model->Evaluate(model->Parameters(), 0, 1);
+      model->Gradient(model->Parameters(), 0, gradient, 1);
+      return error;
+    }
+
+    arma::mat& Parameters() { return model->Parameters(); }
+
+    FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>* model;
+    arma::mat input, target;
+  } function;
+
+  BOOST_REQUIRE_LE(CheckGradient(function), 1e-4);
+}
+
 /**
  * Jacobian MultiplyConstant module test.
  */

From 00e2c9f355079c7fd31e54dc24fe74a88b4385b6 Mon Sep 17 00:00:00 2001
From: manthan-r-sheth <manthanrsheth96@gmail.com>
Date: Wed, 4 Apr 2018 01:47:34 +0530
Subject: [PATCH 40/67] update head

---
 src/mlpack/tests/ann_layer_test.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp
index 57b71cb015..6335351bde 100644
--- a/src/mlpack/tests/ann_layer_test.cpp
+++ b/src/mlpack/tests/ann_layer_test.cpp
@@ -647,8 +647,9 @@ BOOST_AUTO_TEST_CASE(GradientFlexibleReLULayerTest)
       input = arma::randn(10, 1);
       target = arma::mat("1");
 
-      model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>(
-        input, target);
+      model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
+      model->Predictors() = input;
+      model->Responses() = target;
       model->Add<Linear<> >(10, 2);
       model->Add<FlexibleReLU<> >(0.05);
       model->Add<LogSoftMax<> >();

From 420084c883897b3f370ab5d57de4f7d700a0b388 Mon Sep 17 00:00:00 2001
From: Projyal Dev <projyal@gmail.com>
Date: Wed, 4 Apr 2018 15:37:31 +0530
Subject: [PATCH 41/67] Review comments. Removed some code comments.

---
 src/mlpack/methods/ann/rnn_impl.hpp         |  6 ++--
 src/mlpack/tests/recurrent_network_test.cpp | 37 +++++++++++----------
 2 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/src/mlpack/methods/ann/rnn_impl.hpp b/src/mlpack/methods/ann/rnn_impl.hpp
index 5e5bef0db0..04c1ede206 100644
--- a/src/mlpack/methods/ann/rnn_impl.hpp
+++ b/src/mlpack/methods/ann/rnn_impl.hpp
@@ -201,7 +201,7 @@ double RNN<OutputLayerType, InitializationRuleType, CustomLayers...>::Evaluate(
   ResetCells();
 
   double performance = 0;
-  size_t respSeq = 0;
+  size_t responseSeq = 0;
 
   for (size_t seqNum = 0; seqNum < rho; ++seqNum)
   {
@@ -211,7 +211,7 @@ double RNN<OutputLayerType, InitializationRuleType, CustomLayers...>::Evaluate(
     Forward(std::move(stepData));
     if (!single)
     {
-      respSeq = seqNum;
+      responseSeq = seqNum;
     }
 
     if (!deterministic)
@@ -225,7 +225,7 @@ double RNN<OutputLayerType, InitializationRuleType, CustomLayers...>::Evaluate(
 
     performance += outputLayer.Forward(std::move(boost::apply_visitor(
         outputParameterVisitor, network.back())),
-        std::move(arma::mat(responses.slice(respSeq).colptr(begin),
+        std::move(arma::mat(responses.slice(responseSeq).colptr(begin),
             responses.n_rows, batchSize, false, true)));
   }
 
diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp
index 25cd55c3c4..e7f95d0ebb 100644
--- a/src/mlpack/tests/recurrent_network_test.cpp
+++ b/src/mlpack/tests/recurrent_network_test.cpp
@@ -1108,10 +1108,15 @@ BOOST_AUTO_TEST_CASE(CustomRecursiveReberGrammarTest)
  * @param normalize Whether to normalise the data. This may be required for some
  * layers like LSTM. Default is true.
  */
-void GenerateNoisySinRNN(arma::cube& data, arma::cube& labels, size_t rho,
-                         size_t outputSteps = 1, const int dataPoints = 100,
-                         const double gain = 1.0, const int freq = 10,
-                         const double phase = 0, const int noisePercent = 20,
+void GenerateNoisySinRNN(arma::cube& data,
+                         arma::cube& labels,
+                         size_t rho,
+                         size_t outputSteps = 1,
+                         const int dataPoints = 100,
+                         const double gain = 1.0,
+                         const int freq = 10,
+                         const double phase = 0,
+                         const int noisePercent = 20,
                          const double numCycles = 6.0,
                          const bool   normalize = true)
 {
@@ -1125,11 +1130,12 @@ void GenerateNoisySinRNN(arma::cube& data, arma::cube& labels, size_t rho,
   {
     points += rho - r + outputSteps;
   }
-  arma::colvec    x(points);
-  int             i = 0;
-  double          interval = numCycles / freq / points;
-
-  RandomSeed(20);
+  arma::colvec x(points);
+  int i = 0;
+  double interval = numCycles / freq / points;
+  uint64_t timeSeed =
+      std::chrono::high_resolution_clock::now().time_since_epoch().count();
+  RandomSeed(timeSeed);
   x.for_each([&i, gain, freq, phase, noisePercent, interval]
              (arma::colvec::elem_type& val) {
     double t = interval * (i++);
@@ -1145,7 +1151,7 @@ void GenerateNoisySinRNN(arma::cube& data, arma::cube& labels, size_t rho,
   size_t n_columns = y.n_elem / rho;
   data = arma::cube(1, n_columns, rho);
   labels = arma::cube(outputSteps, n_columns, 1);
-  for (int i = 0; i < n_columns; ++i)
+  for (size_t i = 0; i < n_columns; ++i)
   {
     data.tube(0, i) = y.rows(i * rho, i * rho + rho - 1);
     labels.subcube(0, i, 0, outputSteps - 1, i, 0) =
@@ -1167,11 +1173,6 @@ double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100)
   net.Add<LinearNoBias<> >(1, hiddenUnits);
   net.Add<LSTM<> >(hiddenUnits, 1);
 
-  /*
-   * stepSize = 0.05, batchSize = 100, alpha = 0.9, epsilon = 1e-5,
-   * maxiterations = 4000, tolerance = 1e-5
-   *
-   */
   RMSProp opt(0.05, 100, 0.9, 1e-08, 50000, 1e-5);
 
   // Generate data
@@ -1180,8 +1181,8 @@ double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100)
   GenerateNoisySinRNN(data, labels, rho, 1, 20000, 1.0, 200, 0.0, 45, 20);
 
   // Break into training and test sets. Simply split along columns.
-  size_t     trainCols = data.n_cols * 0.8; // Take 20% out for testing.
-  size_t     testCols = data.n_cols - trainCols;
+  size_t trainCols = data.n_cols * 0.8; // Take 20% out for testing.
+  size_t testCols = data.n_cols - trainCols;
   arma::cube testData =
       data.subcube(0, data.n_cols - testCols, 0, data.n_rows - 1,
                    data.n_cols - 1, data.n_slices - 1);
@@ -1189,7 +1190,7 @@ double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100)
       labels.subcube(0, labels.n_cols - testCols, 0, labels.n_rows - 1,
                      labels.n_cols - 1, labels.n_slices - 1);
 
-  for (int i = 0; i < numEpochs; ++i)
+  for (size_t i = 0; i < numEpochs; ++i)
   {
     net.Train(data.subcube(0, 0, 0, data.n_rows - 1, trainCols - 1,
                            data.n_slices - 1),

From 33b0d703c297e27ea83604354fe768054819190f Mon Sep 17 00:00:00 2001
From: Projyal Dev <projyal@gmail.com>
Date: Wed, 4 Apr 2018 15:43:44 +0530
Subject: [PATCH 42/67] Removed commented code.

---
 src/mlpack/tests/recurrent_network_test.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp
index e7f95d0ebb..2417defd7f 100644
--- a/src/mlpack/tests/recurrent_network_test.cpp
+++ b/src/mlpack/tests/recurrent_network_test.cpp
@@ -1204,8 +1204,6 @@ double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100)
 
   // Take slice rho only.
   arma::mat actualPred = prediction.slice(rho - 1);
-  // actualPred.print( "Prediction:" );
-  // testLabels.print( "Actual" );
   double error =
       arma::mean(arma::mean(arma::square(actualPred - testLabels.slice(0))));
   return error;

From e1dc3095f16d8400396c95ffd2dd4207148c1979 Mon Sep 17 00:00:00 2001
From: manthan-r-sheth <manthanrsheth96@gmail.com>
Date: Wed, 4 Apr 2018 18:30:17 +0530
Subject: [PATCH 43/67] update test to uniform distribution

---
 src/mlpack/tests/ann_layer_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp
index 6335351bde..edac8f5dec 100644
--- a/src/mlpack/tests/ann_layer_test.cpp
+++ b/src/mlpack/tests/ann_layer_test.cpp
@@ -644,7 +644,7 @@ BOOST_AUTO_TEST_CASE(GradientFlexibleReLULayerTest)
   {
     GradientFunction()
     {
-      input = arma::randn(10, 1);
+      input = arma::randu(10, 1);
       target = arma::mat("1");
 
       model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();

From 8a22bc04c831a8e009725781f87ef77753d365c0 Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Thu, 5 Apr 2018 02:16:27 +0530
Subject: [PATCH 44/67] Modified according to style guidelines

---
 src/mlpack/methods/ann/layer/dropout.hpp      | 36 +++++++++----------
 src/mlpack/methods/ann/layer/dropout_impl.hpp | 16 ++++-----
 2 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/dropout.hpp b/src/mlpack/methods/ann/layer/dropout.hpp
index 3614f00680..aecb9bab24 100644
--- a/src/mlpack/methods/ann/layer/dropout.hpp
+++ b/src/mlpack/methods/ann/layer/dropout.hpp
@@ -47,11 +47,10 @@ namespace ann /** Artificial Neural Network. */ {
  * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
  */
-template<
-                typename InputDataType = arma::mat,
-                typename OutputDataType = arma::mat
-        >
-class Dropout {
+template<typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat>
+class Dropout
+{
  public:
   /**
    * Create the Dropout object using the specified ratio parameter.
@@ -67,7 +66,7 @@ class Dropout {
    * @param output Resulting output activation.
    */
   template<typename eT>
-  void Forward(const arma::Mat<eT> &&input, arma::Mat<eT> &&output);
+  void Forward(const arma::Mat<eT>&& input, arma::Mat<eT>&& output);
 
   /**
    * Ordinary feed backward pass of the dropout layer.
@@ -77,39 +76,40 @@ class Dropout {
    * @param g The calculated gradient.
    */
   template<typename eT>
-  void Backward(const arma::Mat<eT> && /* input */,
-                arma::Mat<eT> &&gy,
-                arma::Mat<eT> &&g);
+  void Backward(const arma::Mat<eT>&& /* input */,
+                arma::Mat<eT>&& gy,
+                arma::Mat<eT>&& g);
 
   //! Get the input parameter.
-  InputDataType const &InputParameter() const { return inputParameter;  }
+  InputDataType const& InputParameter() const { return inputParameter;  }
 
   //! Modify the input parameter.
-  InputDataType &InputParameter() { return inputParameter; }
+  InputDataType& InputParameter() { return inputParameter; }
 
   //! Get the output parameter.
-  OutputDataType const &OutputParameter() const { return outputParameter; }
+  OutputDataType const& OutputParameter() const { return outputParameter; }
 
   //! Modify the output parameter.
-  OutputDataType &OutputParameter() { return outputParameter; }
+  OutputDataType& OutputParameter() { return outputParameter; }
 
   //! Get the detla.
-  OutputDataType const &Delta() const { return delta; }
+  OutputDataType const& Delta() const { return delta; }
 
   //! Modify the delta.
-  OutputDataType &Delta() { return delta; }
+  OutputDataType& Delta() { return delta; }
 
   //! The value of the deterministic parameter.
   bool Deterministic() const { return deterministic; }
 
   //! Modify the value of the deterministic parameter.
-  bool &Deterministic() { return deterministic; }
+  bool& Deterministic() { return deterministic; }
 
   //! The probability of setting a value to zero.
   double Ratio() const { return ratio; }
 
   //! Modify the probability of setting a value to zero.
-  void Ratio(const double r) {
+  void Ratio(const double r)
+  {
     ratio = r;
     scale = 1.0 / (1.0 - ratio);
   }
@@ -118,7 +118,7 @@ class Dropout {
    * Serialize the layer.
    */
   template<typename Archive>
-  void serialize(Archive &ar, const unsigned int /* version */);
+  void serialize(Archive& ar, const unsigned int /* version */);
 
  private:
   //! Locally-stored delta object.
diff --git a/src/mlpack/methods/ann/layer/dropout_impl.hpp b/src/mlpack/methods/ann/layer/dropout_impl.hpp
index 42ccfbf47e..a80c6da618 100644
--- a/src/mlpack/methods/ann/layer/dropout_impl.hpp
+++ b/src/mlpack/methods/ann/layer/dropout_impl.hpp
@@ -32,16 +32,16 @@ Dropout<InputDataType, OutputDataType>::Dropout(
 template<typename InputDataType, typename OutputDataType>
 template<typename eT>
 void Dropout<InputDataType, OutputDataType>::Forward(
-    const arma::Mat<eT> &&input,
-    arma::Mat<eT> &&output)
+    const arma::Mat<eT>&& input,
+    arma::Mat<eT>&& output)
 {
   // The dropout mask will not be multiplied in the deterministic mode
   // (during testing).
   if (deterministic) {
     output = input;
   } else {
-    // Scale with input / (1 - ratio) and set values to zero
-    // with probability 'ratio'.
+    // Scale with input / (1 - ratio) and set values to zero with probability
+    // 'ratio'.
 
     mask = arma::randu<arma::Mat<eT> >(input.n_rows, input.n_cols);
     mask.transform([&](double val) { return (val > ratio); });
@@ -52,9 +52,9 @@ void Dropout<InputDataType, OutputDataType>::Forward(
 template<typename InputDataType, typename OutputDataType>
 template<typename eT>
 void Dropout<InputDataType, OutputDataType>::Backward(
-    const arma::Mat<eT> && /* input */,
-    arma::Mat<eT> &&gy,
-    arma::Mat<eT> &&g)
+    const arma::Mat<eT>&& /* input */,
+    arma::Mat<eT>&& gy,
+    arma::Mat<eT>&& g)
 {
   g = gy % mask * scale;
 }
@@ -62,7 +62,7 @@ void Dropout<InputDataType, OutputDataType>::Backward(
 template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
 void Dropout<InputDataType, OutputDataType>::serialize(
-    Archive &ar,
+    Archive& ar,
     const unsigned int /* version */)
 {
   ar & BOOST_SERIALIZATION_NVP(ratio);

From da21d21960d186e11d79982586f90c60ed546037 Mon Sep 17 00:00:00 2001
From: manthan-r-sheth <manthanrsheth96@gmail.com>
Date: Thu, 5 Apr 2018 02:54:20 +0530
Subject: [PATCH 45/67] update gradient method

---
 src/mlpack/methods/ann/layer/flexible_relu_impl.hpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index 2fa1c5dca6..2e7b66bc50 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -68,8 +68,7 @@ void FlexibleReLU<InputDataType, OutputDataType>::Gradient(
   {
     gradient = arma::zeros<arma::Mat<eT>>(1, 1);
   }
-  gradient(0) = arma::accu(error % arma::clamp(input, -DBL_MAX, 0.0))
-      / input.n_cols;
+  gradient(0) = 1.0;
 }
 
 

From 45e82cdac68cd1091c543cf27b848f0f3e63354a Mon Sep 17 00:00:00 2001
From: manthan-r-sheth <manthanrsheth96@gmail.com>
Date: Thu, 5 Apr 2018 10:50:14 +0530
Subject: [PATCH 46/67] gradient update

---
 src/mlpack/methods/ann/layer/flexible_relu_impl.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index 2e7b66bc50..097dbc0a5f 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -68,7 +68,7 @@ void FlexibleReLU<InputDataType, OutputDataType>::Gradient(
   {
     gradient = arma::zeros<arma::Mat<eT>>(1, 1);
   }
-  gradient(0) = 1.0;
+  gradient(0) = arma::accu(error) / input.n_cols;
 }
 
 

From 08fdb6c665c08286c3a055a8b8c2b994678fd065 Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Thu, 5 Apr 2018 23:46:19 +0530
Subject: [PATCH 47/67] More style fixes

---
 src/mlpack/methods/ann/layer/dropout.hpp      | 2 +-
 src/mlpack/methods/ann/layer/dropout_impl.hpp | 8 +++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/dropout.hpp b/src/mlpack/methods/ann/layer/dropout.hpp
index aecb9bab24..d1f90dd124 100644
--- a/src/mlpack/methods/ann/layer/dropout.hpp
+++ b/src/mlpack/methods/ann/layer/dropout.hpp
@@ -48,7 +48,7 @@ namespace ann /** Artificial Neural Network. */ {
  *         arma::sp_mat or arma::cube).
  */
 template<typename InputDataType = arma::mat,
-    typename OutputDataType = arma::mat>
+         typename OutputDataType = arma::mat>
 class Dropout
 {
  public:
diff --git a/src/mlpack/methods/ann/layer/dropout_impl.hpp b/src/mlpack/methods/ann/layer/dropout_impl.hpp
index a80c6da618..ccac9588f8 100644
--- a/src/mlpack/methods/ann/layer/dropout_impl.hpp
+++ b/src/mlpack/methods/ann/layer/dropout_impl.hpp
@@ -37,12 +37,14 @@ void Dropout<InputDataType, OutputDataType>::Forward(
 {
   // The dropout mask will not be multiplied in the deterministic mode
   // (during testing).
-  if (deterministic) {
+  if (deterministic)
+  {
     output = input;
-  } else {
+  }
+  else
+  {
     // Scale with input / (1 - ratio) and set values to zero with probability
     // 'ratio'.
-
     mask = arma::randu<arma::Mat<eT> >(input.n_rows, input.n_cols);
     mask.transform([&](double val) { return (val > ratio); });
     output = input % mask * scale;

From c0c4e78588a5c70ffe302c1a2aed80529e4b71f3 Mon Sep 17 00:00:00 2001
From: Prabhat <prabhatsharma7298@gmail.com>
Date: Fri, 6 Apr 2018 03:14:09 +0530
Subject: [PATCH 48/67] Fix citation style

---
 .../ann/init_rules/lecun_normal_init.hpp      | 23 ++++++++++---------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
index 4a1bc9ce04..41ad1e9121 100644
--- a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
@@ -1,6 +1,7 @@
 /**
  * @file lecun_normal_init.hpp
- * @author Dakshit Agrawal and Prabhat Sharma
+ * @author Dakshit Agrawal
+ * @author Prabhat Sharma
  *
  * Intialization rule given by Lecun et. al. for neural networks and
  * also mentioned in Self Normalizing Networks.
@@ -28,18 +29,18 @@ namespace ann /** Artificial Neural Network. */ {
  *
  * @code
  * @inproceedings{conf/nips/KlambauerUMH17,
- * title = {Self-Normalizing Neural Networks.},
+ * title  = {Self-Normalizing Neural Networks.},
  * author = {Klambauer, Günter and Unterthiner, Thomas
- * and Mayr, Andreas and Hochreiter, Sepp},
- * pages = {972-981},
- * year = 2017}
+ *           and Mayr, Andreas and Hochreiter, Sepp},
+ * pages  = {972-981},
+ * year   = 2017}
  *
  * @inproceedings{LeCun:1998:EB:645754.668382,
- * title = {Efficient BackProp},
+ * title  = {Efficient BackProp},
  * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B.
- * and M\"{u}ller, Klaus-Robert},
- * year = {1998},
- * pages = {9--50}}
+ *          and M\"{u}ller, Klaus-Robert},
+ * year   = {1998},
+ * pages  = {9--50}}
  * @endcode
  *
 */
@@ -69,7 +70,7 @@ class LecunNormalInitialization
   {
     // He initialization rule says to initialize weights with random
     // values taken from a gaussian distribution with mean = 0 and
-    // standard deviation = sqrt(1/rows), i.e. variance = (1/rows).
+    // standard deviation = sqrt(1 / rows), i.e. variance = (1 / rows).
     const double variance = 1.0 / ((double) rows);
 
     if (W.is_empty())
@@ -78,7 +79,7 @@ class LecunNormalInitialization
     }
 
     // Multipling a random variable X with variance V(X) by some factor c,
-    // then the variance V(cX) = (c^2)* V(X).
+    // then the variance V(cX) = (c ^ 2) * V(X).
     W.imbue( [&]() { return sqrt(variance) * arma::randn(); } );
   }
 

From 36f23770e186d0b5608254418572e3b056c68d3b Mon Sep 17 00:00:00 2001
From: manthan-r-sheth <manthanrsheth96@gmail.com>
Date: Fri, 6 Apr 2018 16:18:26 +0530
Subject: [PATCH 49/67] optimised gradient

---
 src/mlpack/methods/ann/layer/flexible_relu_impl.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index 097dbc0a5f..13106f1042 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -66,7 +66,7 @@ void FlexibleReLU<InputDataType, OutputDataType>::Gradient(
 {
   if (gradient.n_elem == 0)
   {
-    gradient = arma::zeros<arma::Mat<eT>>(1, 1);
+    gradient.set_size(1, 1);
   }
   gradient(0) = arma::accu(error) / input.n_cols;
 }

From 914a656ced46034f233cc814e5ac6a24775774d6 Mon Sep 17 00:00:00 2001
From: Projyal Dev <projyal@gmail.com>
Date: Sat, 7 Apr 2018 16:48:04 +0530
Subject: [PATCH 50/67] Changed the error calculation to reflect the error more
 accurately. Also reduced the test data for tests to complete faster.

---
 src/mlpack/tests/recurrent_network_test.cpp | 22 +++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp
index 2417defd7f..bb49d86ced 100644
--- a/src/mlpack/tests/recurrent_network_test.cpp
+++ b/src/mlpack/tests/recurrent_network_test.cpp
@@ -1171,14 +1171,15 @@ double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100)
 {
   RNN<MeanSquaredError<> > net(rho, true);
   net.Add<LinearNoBias<> >(1, hiddenUnits);
-  net.Add<LSTM<> >(hiddenUnits, 1);
+  net.Add<LSTM<> >(hiddenUnits, hiddenUnits);
+  net.Add<LinearNoBias<> >(hiddenUnits, 1);
 
-  RMSProp opt(0.05, 100, 0.9, 1e-08, 50000, 1e-5);
+  RMSProp opt(0.005, 100, 0.9, 1e-08, 50000, 1e-5);
 
   // Generate data
   arma::cube data;
   arma::cube labels;
-  GenerateNoisySinRNN(data, labels, rho, 1, 20000, 1.0, 200, 0.0, 45, 20);
+  GenerateNoisySinRNN(data, labels, rho, 1, 2000, 20.0, 200, 0.0, 45, 20);
 
   // Break into training and test sets. Simply split along columns.
   size_t trainCols = data.n_cols * 0.8; // Take 20% out for testing.
@@ -1202,10 +1203,15 @@ double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100)
   arma::cube prediction;
   net.Predict(testData, prediction);
 
-  // Take slice rho only.
-  arma::mat actualPred = prediction.slice(rho - 1);
-  double error =
-      arma::mean(arma::mean(arma::square(actualPred - testLabels.slice(0))));
+  // The prediction must really follow the test data. So convert both the test
+  // data and the pediction to vectors and compare the two.
+  arma::colvec testVector = arma::vectorise(testData);
+  arma::colvec predVector = arma::vectorise(prediction);
+  //Adjust the vectors for comparison, as the prediction is one step ahead.
+  testVector = testVector.rows(1, testVector.n_rows - 1);
+  predVector = predVector.rows(0, predVector.n_rows - 2);
+  double error = std::sqrt(arma::sum(arma::square(testVector - predVector))) /
+                 testVector.n_rows;
   return error;
 }
 
@@ -1214,7 +1220,7 @@ double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100)
  */
 BOOST_AUTO_TEST_CASE(MultiTimestepTest)
 {
-  double err = RNNSineTest(112, 10, 10);
+  double err = RNNSineTest(4, 10, 20);
   BOOST_REQUIRE_LE(err, 1e-02);
 }
 

From 476302a32792152159809449289ad6b2d2ac0429 Mon Sep 17 00:00:00 2001
From: manthan-r-sheth <manthanrsheth96@gmail.com>
Date: Sat, 7 Apr 2018 23:00:35 +0530
Subject: [PATCH 51/67] update gradient for positive parameters

---
 src/mlpack/tests/ann_layer_test.cpp | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp
index edac8f5dec..81097c8fbe 100644
--- a/src/mlpack/tests/ann_layer_test.cpp
+++ b/src/mlpack/tests/ann_layer_test.cpp
@@ -644,13 +644,15 @@ BOOST_AUTO_TEST_CASE(GradientFlexibleReLULayerTest)
   {
     GradientFunction()
     {
-      input = arma::randu(10, 1);
+      input = arma::randu(2, 1);
       target = arma::mat("1");
 
-      model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
+      model = new FFN<NegativeLogLikelihood<>, RandomInitialization>(
+        NegativeLogLikelihood<>(), RandomInitialization(0.1, 0.5));
+
       model->Predictors() = input;
       model->Responses() = target;
-      model->Add<Linear<> >(10, 2);
+      model->Add<LinearNoBias<> >(2, 5);
       model->Add<FlexibleReLU<> >(0.05);
       model->Add<LogSoftMax<> >();
     }
@@ -670,7 +672,7 @@ BOOST_AUTO_TEST_CASE(GradientFlexibleReLULayerTest)
 
     arma::mat& Parameters() { return model->Parameters(); }
 
-    FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>* model;
+    FFN<NegativeLogLikelihood<>, RandomInitialization>* model;
     arma::mat input, target;
   } function;
 

From 96480b97a85ec4c9491580f943bde67c9128c0c8 Mon Sep 17 00:00:00 2001
From: Prabhat <prabhatsharma7298@gmail.com>
Date: Sun, 8 Apr 2018 20:57:55 +0530
Subject: [PATCH 52/67] Fixed minor style issue

Doxygen authors changed to author
---
 src/mlpack/methods/ann/init_rules/he_init.hpp | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp
index b0ee3c62b7..d555464402 100644
--- a/src/mlpack/methods/ann/init_rules/he_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/he_init.hpp
@@ -1,6 +1,7 @@
 /**
  * @file he_init.hpp
- * @authors Dakshit Agrawal and Prabhat Sharma
+ * @author Dakshit Agrawal
+ * @author Prabhat Sharma
  *
  * Intialization rule given by He et. al. for neural networks. The He
  * initialization initializes weights of the neural network to better
@@ -31,12 +32,12 @@ namespace ann /** Artificial Neural Network. */ {
  *
  * @code
  * @article{He2015DelvingDI,
- * title={Delving Deep into Rectifiers: Surpassing Human-Level Performance
- * on ImageNet Classification},
- * author={Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun},
- * journal={2015 IEEE International Conference on Computer Vision (ICCV)},
- * year={2015},
- * pages={1026-1034}}
+ * title   = {Delving Deep into Rectifiers: Surpassing Human-Level Performance
+ *            on ImageNet Classification},
+ * author  = {Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun},
+ * journal = {2015 IEEE International Conference on Computer Vision (ICCV)},
+ * year    = {2015},
+ * pages   = {1026-1034}}
  * @endcode
  *
  */

From c1d7a5864d27ced0ac01445b38900837b33c65a5 Mon Sep 17 00:00:00 2001
From: Projyal Dev <projyal@gmail.com>
Date: Mon, 9 Apr 2018 08:45:24 +0530
Subject: [PATCH 53/67] Style correction.

---
 src/mlpack/tests/recurrent_network_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp
index bb49d86ced..f170807f39 100644
--- a/src/mlpack/tests/recurrent_network_test.cpp
+++ b/src/mlpack/tests/recurrent_network_test.cpp
@@ -1207,7 +1207,7 @@ double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100)
   // data and the pediction to vectors and compare the two.
   arma::colvec testVector = arma::vectorise(testData);
   arma::colvec predVector = arma::vectorise(prediction);
-  //Adjust the vectors for comparison, as the prediction is one step ahead.
+  // Adjust the vectors for comparison, as the prediction is one step ahead.
   testVector = testVector.rows(1, testVector.n_rows - 1);
   predVector = predVector.rows(0, predVector.n_rows - 2);
   double error = std::sqrt(arma::sum(arma::square(testVector - predVector))) /

From 7c603cf7ff7c9562571e71d08ad648d9a04e562b Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Mon, 9 Apr 2018 20:55:18 +0530
Subject: [PATCH 54/67] Removed unnecessary spaces provided

---
 src/mlpack/methods/ann/layer/dropout.hpp | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/dropout.hpp b/src/mlpack/methods/ann/layer/dropout.hpp
index d1f90dd124..a388820c71 100644
--- a/src/mlpack/methods/ann/layer/dropout.hpp
+++ b/src/mlpack/methods/ann/layer/dropout.hpp
@@ -82,25 +82,21 @@ class Dropout
 
   //! Get the input parameter.
   InputDataType const& InputParameter() const { return inputParameter;  }
-
   //! Modify the input parameter.
   InputDataType& InputParameter() { return inputParameter; }
 
   //! Get the output parameter.
   OutputDataType const& OutputParameter() const { return outputParameter; }
-
   //! Modify the output parameter.
   OutputDataType& OutputParameter() { return outputParameter; }
 
   //! Get the detla.
   OutputDataType const& Delta() const { return delta; }
-
   //! Modify the delta.
   OutputDataType& Delta() { return delta; }
 
   //! The value of the deterministic parameter.
   bool Deterministic() const { return deterministic; }
-
   //! Modify the value of the deterministic parameter.
   bool& Deterministic() { return deterministic; }
 

From 6f084f8f0ad4fca6035615680be6c2d5693c4215 Mon Sep 17 00:00:00 2001
From: luffy <rajrohan1108@gmail.com>
Date: Tue, 10 Apr 2018 19:07:12 +0530
Subject: [PATCH 55/67] Header file : Adam  update to vanilla update

---
 src/mlpack/tests/async_learning_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/tests/async_learning_test.cpp b/src/mlpack/tests/async_learning_test.cpp
index feb16e4ce2..42acc75389 100644
--- a/src/mlpack/tests/async_learning_test.cpp
+++ b/src/mlpack/tests/async_learning_test.cpp
@@ -16,7 +16,7 @@
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/reinforcement_learning/async_learning.hpp>
 #include <mlpack/methods/reinforcement_learning/environment/cart_pole.hpp>
-#include <mlpack/core/optimizers/adam/adam_update.hpp>
+#include <mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp>
 #include <mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp>
 #include <mlpack/methods/reinforcement_learning/policy/aggregated_policy.hpp>
 #include <mlpack/methods/reinforcement_learning/training_config.hpp>

From b4113010184356cec2da94a49893be041876c423 Mon Sep 17 00:00:00 2001
From: luffy <rajrohan1108@gmail.com>
Date: Tue, 10 Apr 2018 19:12:16 +0530
Subject: [PATCH 56/67] name

---
 src/mlpack/tests/async_learning_test.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/mlpack/tests/async_learning_test.cpp b/src/mlpack/tests/async_learning_test.cpp
index 42acc75389..cf0da9b526 100644
--- a/src/mlpack/tests/async_learning_test.cpp
+++ b/src/mlpack/tests/async_learning_test.cpp
@@ -1,6 +1,7 @@
 /**
  * @file async_learning_test.hpp
  * @author Shangtong Zhang
+ * @author Rohan Raj
  *
  * Test for async deep RL methods.
  *

From 32a9e58c1a277a264555080a45d48d9e150f2cdc Mon Sep 17 00:00:00 2001
From: manthan-r-sheth <manthanrsheth96@gmail.com>
Date: Wed, 11 Apr 2018 01:00:28 +0530
Subject: [PATCH 57/67] improve readability

---
 src/mlpack/methods/ann/layer/flexible_relu_impl.hpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index 13106f1042..1a3382d2f7 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -53,8 +53,7 @@ void FlexibleReLU<InputDataType, OutputDataType>::Backward(
 {
   DataType derivative;
   //! Compute the first derivative of FlexibleReLU function.
-  derivative = arma::sign(input);
-  derivative.elem(arma::find(derivative < 0.0)) += 1;
+  derivative = arma::clamp(arma::sign(input), 0.0, 1.0);
   g = gy % derivative;
 }
 

From cb1d2d4b88fa00544637cc193f8c4d29b27defbe Mon Sep 17 00:00:00 2001
From: Prabhat <prabhatsharma7298@gmail.com>
Date: Wed, 11 Apr 2018 02:48:11 +0530
Subject: [PATCH 58/67] Fix citation style

---
 src/mlpack/methods/ann/init_rules/he_init.hpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp
index d555464402..822fb5a81e 100644
--- a/src/mlpack/methods/ann/init_rules/he_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/he_init.hpp
@@ -32,12 +32,12 @@ namespace ann /** Artificial Neural Network. */ {
  *
  * @code
  * @article{He2015DelvingDI,
- * title   = {Delving Deep into Rectifiers: Surpassing Human-Level Performance
- *            on ImageNet Classification},
- * author  = {Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun},
- * journal = {2015 IEEE International Conference on Computer Vision (ICCV)},
- * year    = {2015},
- * pages   = {1026-1034}}
+ *   title   = {Delving Deep into Rectifiers: Surpassing Human-Level Performance
+ *              on ImageNet Classification},
+ *   author  = {Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun},
+ *   journal = {2015 IEEE International Conference on Computer Vision (ICCV)},
+ *   year    = {2015},
+ *   pages   = {1026-1034}}
  * @endcode
  *
  */

From 617b9e529c046cefb577c2125f06f46719971271 Mon Sep 17 00:00:00 2001
From: Marcus Edel <marcus.edel@fu-berlin.de>
Date: Wed, 11 Apr 2018 23:25:37 +0200
Subject: [PATCH 59/67] Minor style fixes (indentation, comments, parameter
 rename).

---
 .../methods/ann/layer/flexible_relu.hpp       | 23 +++++++++----------
 .../methods/ann/layer/flexible_relu_impl.hpp  | 20 ++++++++--------
 src/mlpack/tests/ann_layer_test.cpp           |  2 +-
 3 files changed, 22 insertions(+), 23 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/flexible_relu.hpp b/src/mlpack/methods/ann/layer/flexible_relu.hpp
index 7b08fb3a8f..a9c837e022 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu.hpp
@@ -6,7 +6,7 @@
  * Definition of FlexibleReLU layer as described by
  * Suo Qiu, Xiangmin Xu and Bolun Cai in
  * "FReLU: Flexible Rectified Linear Units for Improving Convolutional
- *   Neural Networks", 2018
+ * Neural Networks", 2018
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -22,7 +22,7 @@ namespace mlpack {
 namespace ann /**Artificial Neural Network*/ {
 
 /**
- *The FlexibleReLU activation function, defined by
+ * The FlexibleReLU activation function, defined by
  *
  * @f{eqnarray*}{
  * f(x) &=& \max(0,x)+alpha \\
@@ -32,7 +32,7 @@ namespace ann /**Artificial Neural Network*/ {
  * 	   0 & : x \le 0
  * 	 \end{array}
  * \right
- *@f}
+ * @f}
  *
  * For more information, read the following paper:
  *
@@ -47,11 +47,10 @@ namespace ann /**Artificial Neural Network*/ {
  * }
  * @endcode
  *
- *@tparam InputDataType Type of the input data (arma::colvec, arma::mar,
- *        arma::sp_mat or arma::cube)
- *@tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
- *        arma::sp_mat or arma::cube)
- *
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mar,
+ *         arma::sp_mat or arma::cube)
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube)
  */
 template <
     typename InputDataType = arma::mat,
@@ -64,15 +63,15 @@ class FlexibleReLU
    *
    * Create the FlexibleReLU object using the specified parameters.
    * The non zero parameter can be adjusted by specifying the parameter
-   * alpha which controls the range of the relu function. ( Default alpha = 0)
+   * alpha which controls the range of the relu function. (Default alpha = 0)
    * This parameter is trainable.
    *
-   *@param alpha Parameter for adjusting the range of the relu function.
+   * @param alpha Parameter for adjusting the range of the relu function.
    *
    */
-  FlexibleReLU(const double userAlpha = 0);
+  FlexibleReLU(const double alpha = 0);
 
-  /*
+  /**
    * Reset the layer parameter.
    */
   void Reset();
diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index 1a3382d2f7..52ab56f498 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -5,9 +5,9 @@
  *
  * Implementation of FlexibleReLU layer as described by
  * Suo Qiu, Xiangmin Xu and Bolun Cai in
- * "FReLU: Flexible Rectified Linear Units for Improving Convolutional 
+ * "FReLU: Flexible Rectified Linear Units for Improving Convolutional
  *  Neural Networks", 2018
- *  
+ *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
  * 3-clause BSD license along with mlpack.  If not, see
@@ -25,10 +25,10 @@ namespace ann /** Artificial Neural Network. */ {
 
 template<typename InputDataType, typename OutputDataType>
 FlexibleReLU<InputDataType, OutputDataType>::FlexibleReLU(
-    const double userAlpha) : userAlpha(userAlpha)
+    const double alpha) : userAlpha(alpha)
 {
-  alpha.set_size(1, 1);
-  alpha(0) = userAlpha;
+  this->alpha.set_size(1, 1);
+  this->alpha(0) = userAlpha;
 }
 
 template<typename InputDataType, typename OutputDataType>
@@ -51,22 +51,22 @@ template<typename DataType>
 void FlexibleReLU<InputDataType, OutputDataType>::Backward(
     const DataType&& input, DataType&& gy, DataType&& g)
 {
-  DataType derivative;
   //! Compute the first derivative of FlexibleReLU function.
-  derivative = arma::clamp(arma::sign(input), 0.0, 1.0);
-  g = gy % derivative;
+  g = gy % arma::clamp(arma::sign(input), 0.0, 1.0);
 }
 
 template<typename InputDataType, typename OutputDataType>
 template<typename eT>
 void FlexibleReLU<InputDataType, OutputDataType>::Gradient(
-  const arma::Mat<eT>&& input, arma::Mat<eT>&& error,
-  arma::Mat<eT>&& gradient)
+    const arma::Mat<eT>&& input,
+    arma::Mat<eT>&& error,
+    arma::Mat<eT>&& gradient)
 {
   if (gradient.n_elem == 0)
   {
     gradient.set_size(1, 1);
   }
+
   gradient(0) = arma::accu(error) / input.n_cols;
 }
 
diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp
index 9494077fd3..d6d1478a82 100644
--- a/src/mlpack/tests/ann_layer_test.cpp
+++ b/src/mlpack/tests/ann_layer_test.cpp
@@ -732,7 +732,7 @@ BOOST_AUTO_TEST_CASE(GradientFlexibleReLULayerTest)
       target = arma::mat("1");
 
       model = new FFN<NegativeLogLikelihood<>, RandomInitialization>(
-        NegativeLogLikelihood<>(), RandomInitialization(0.1, 0.5));
+          NegativeLogLikelihood<>(), RandomInitialization(0.1, 0.5));
 
       model->Predictors() = input;
       model->Responses() = target;

From 2224223b6b0b6e7ccaaefe6368e5d1cfbc8ee5c6 Mon Sep 17 00:00:00 2001
From: luffy <rajrohan1108@gmail.com>
Date: Thu, 12 Apr 2018 09:44:53 +0530
Subject: [PATCH 60/67] Minor changes

---
 src/mlpack/tests/async_learning_test.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/mlpack/tests/async_learning_test.cpp b/src/mlpack/tests/async_learning_test.cpp
index cf0da9b526..c937771164 100644
--- a/src/mlpack/tests/async_learning_test.cpp
+++ b/src/mlpack/tests/async_learning_test.cpp
@@ -1,7 +1,6 @@
 /**
  * @file async_learning_test.hpp
  * @author Shangtong Zhang
- * @author Rohan Raj
  *
  * Test for async deep RL methods.
  *
@@ -36,7 +35,7 @@ BOOST_AUTO_TEST_SUITE(AsyncLearningTest);
 BOOST_AUTO_TEST_CASE(OneStepQLearningTest)
 {
   /**
-   * This is for the Travis CI server, in your own machine you shuold use more
+   * This is for the Travis CI server, in your own machine you should use more
    * threads.
    */
   #ifdef HAS_OPENMP

From 1b7e9301513ed33091e6b14b4a3ffe22fe944ded Mon Sep 17 00:00:00 2001
From: Projyal Dev <projyal@gmail.com>
Date: Fri, 13 Apr 2018 22:56:14 +0530
Subject: [PATCH 61/67] Style changes and removed the RandomSeed call from
 noisy sine generator for the RNN.

---
 src/mlpack/methods/ann/rnn_impl.hpp         |  4 +--
 src/mlpack/tests/recurrent_network_test.cpp | 39 ++++++++++-----------
 2 files changed, 20 insertions(+), 23 deletions(-)

diff --git a/src/mlpack/methods/ann/rnn_impl.hpp b/src/mlpack/methods/ann/rnn_impl.hpp
index 04c1ede206..bc93b5fa89 100644
--- a/src/mlpack/methods/ann/rnn_impl.hpp
+++ b/src/mlpack/methods/ann/rnn_impl.hpp
@@ -290,8 +290,8 @@ void RNN<OutputLayerType, InitializationRuleType, CustomLayers...>::Gradient(
     {
       outputLayer.Backward(std::move(boost::apply_visitor(
           outputParameterVisitor, network.back())),
-          std::move(arma::mat(responses.slice(0).colptr(begin),
-              responses.n_rows, batchSize, false, true)),
+              std::move(arma::mat(responses.slice(0).colptr(begin),
+                  responses.n_rows, batchSize, false, true)),
           std::move(error));
     }
     else
diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp
index f170807f39..0f8fb5b46e 100644
--- a/src/mlpack/tests/recurrent_network_test.cpp
+++ b/src/mlpack/tests/recurrent_network_test.cpp
@@ -1090,23 +1090,23 @@ BOOST_AUTO_TEST_CASE(CustomRecursiveReberGrammarTest)
 
 /**
  * @brief Generates noisy sine wave and outputs the data and the labels that
- * can be used directly for training and testing with RNN.
+ *        can be used directly for training and testing with RNN.
  *
- * @param data    The data points as output
- * @param labels  The expected values as output
- * @param rho     The size of the sequence of each data point
+ * @param data The data points as output
+ * @param labels The expected values as output
+ * @param rho The size of the sequence of each data point
  * @param outputSteps How many output steps to consider for every rho inputs
  * @param dataPoints  The number of generated data points. The actual generated
- * data points may be more than this to adjust to the outputSteps.
- * But at the minimum these many data points will be generated.
- * @param gain    The gain on the amplitude
- * @param freq    The frquency of the sine wave
- * @param phase   The phase shift if any
- * @param noisePercent  The percent noise to induce
- * @param numCycles  How many full size wave cycles required. All the data
- * points will be fit into these cycles.
+ *        data points may be more than this to adjust to the outputSteps. But at
+ *        the minimum these many data points will be generated.
+ * @param gain The gain on the amplitude
+ * @param freq The frquency of the sine wave
+ * @param phase The phase shift if any
+ * @param noisePercent The percent noise to induce
+ * @param numCycles How many full size wave cycles required. All the data
+ *        points will be fit into these cycles.
  * @param normalize Whether to normalise the data. This may be required for some
- * layers like LSTM. Default is true.
+ *        layers like LSTM. Default is true.
  */
 void GenerateNoisySinRNN(arma::cube& data,
                          arma::cube& labels,
@@ -1118,7 +1118,7 @@ void GenerateNoisySinRNN(arma::cube& data,
                          const double phase = 0,
                          const int noisePercent = 20,
                          const double numCycles = 6.0,
-                         const bool   normalize = true)
+                         const bool normalize = true)
 {
   int points = dataPoints;
   int r = dataPoints % rho;
@@ -1133,9 +1133,6 @@ void GenerateNoisySinRNN(arma::cube& data,
   arma::colvec x(points);
   int i = 0;
   double interval = numCycles / freq / points;
-  uint64_t timeSeed =
-      std::chrono::high_resolution_clock::now().time_since_epoch().count();
-  RandomSeed(timeSeed);
   x.for_each([&i, gain, freq, phase, noisePercent, interval]
              (arma::colvec::elem_type& val) {
     double t = interval * (i++);
@@ -1148,10 +1145,10 @@ void GenerateNoisySinRNN(arma::cube& data,
     y = arma::normalise(x);
 
   // Now break this into columns of rho size slices.
-  size_t n_columns = y.n_elem / rho;
-  data = arma::cube(1, n_columns, rho);
-  labels = arma::cube(outputSteps, n_columns, 1);
-  for (size_t i = 0; i < n_columns; ++i)
+  size_t numColumns = y.n_elem / rho;
+  data = arma::cube(1, numColumns, rho);
+  labels = arma::cube(outputSteps, numColumns, 1);
+  for (size_t i = 0; i < numColumns; ++i)
   {
     data.tube(0, i) = y.rows(i * rho, i * rho + rho - 1);
     labels.subcube(0, i, 0, outputSteps - 1, i, 0) =

From e44e13f864b388ead5e8f8c3749d0dae3dd04a62 Mon Sep 17 00:00:00 2001
From: Rohan Raj <rajrohan1108@gmail.com>
Date: Mon, 16 Apr 2018 23:49:47 +0530
Subject: [PATCH 62/67] Acrobat game for Reinforcement learning Environments
 (#1329)

* initial commit

* Added dsdt to the code

* Basic Structure ready
Check for errors

* minor changes

* Added Acrobat

* Comments Edited

* Build Successful

* style fixes

* Style Fixes

* style fixes

* style fixes

* Style

* Style

* Reviews added

* Reviews

* Reviews

* Added Acrobat test
Convergence

* comment edited

* Added my name

* Minor Style Fix

* Add necessary tests

* 50 trials for Acrobat

* Added 50 tests

* Changes in requirement

* Necessary Changes to reduce time for tests

* MInor Style Changes

* Revert Changes  in Acrobat Game
Removed 50 trials

* Style

* Reviews added

* Style Changes
Added reviews

* Comments Added
File is reviewed

* Added name to contributor list
---
 src/mlpack/core.hpp                           |   1 +
 .../environment/CMakeLists.txt                |   1 +
 .../environment/acrobat.hpp                   | 343 ++++++++++++++++++
 src/mlpack/tests/q_learning_test.cpp          |  66 ++++
 src/mlpack/tests/rl_components_test.cpp       |  18 +
 5 files changed, 429 insertions(+)
 create mode 100644 src/mlpack/methods/reinforcement_learning/environment/acrobat.hpp

diff --git a/src/mlpack/core.hpp b/src/mlpack/core.hpp
index 4989323097..f5661bce73 100644
--- a/src/mlpack/core.hpp
+++ b/src/mlpack/core.hpp
@@ -241,6 +241,7 @@
  *   - Tan Jun An <yamidarkxxx@gmail.com>
  *   - Moksh Jain <mokshjn00@gmail.com>
  *   - Manthan-R-Sheth <manthanrsheth96@gmail.com>
+ *   - Rohan Raj <rajrohan1108@gmail.com>
  */
 
 // First, include all of the prerequisites.
diff --git a/src/mlpack/methods/reinforcement_learning/environment/CMakeLists.txt b/src/mlpack/methods/reinforcement_learning/environment/CMakeLists.txt
index 58baa7e14d..6814fbab5d 100644
--- a/src/mlpack/methods/reinforcement_learning/environment/CMakeLists.txt
+++ b/src/mlpack/methods/reinforcement_learning/environment/CMakeLists.txt
@@ -3,6 +3,7 @@
 set(SOURCES
   mountain_car.hpp
   cart_pole.hpp
+  acrobat.hpp
 )
 
 # Add directory name to sources.
diff --git a/src/mlpack/methods/reinforcement_learning/environment/acrobat.hpp b/src/mlpack/methods/reinforcement_learning/environment/acrobat.hpp
new file mode 100644
index 0000000000..bacd9a208f
--- /dev/null
+++ b/src/mlpack/methods/reinforcement_learning/environment/acrobat.hpp
@@ -0,0 +1,343 @@
+/**
+ * @file acrobat.hpp
+ * @author Rohan Raj
+ *
+ * This file is an implementation of Acrobat task:
+ * https://gym.openai.com/envs/Acrobot-v1/
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_RL_ENVIRONMENT_ACROBAT_HPP
+#define MLPACK_METHODS_RL_ENVIRONMENT_ACROBAT_HPP
+
+#include <mlpack/core.hpp>
+
+namespace mlpack{
+namespace rl{
+/**
+ * Implementation of Acrobat game
+ * Acrobot is a 2-link pendulum with only the second joint actuated
+ * Intitially, both links point downwards. The goal is to swing the
+ * end-effector at a height at least the length of one link above the base.
+ * Both links can swing freely and can pass by each other, i.e., they don't
+ * collide when they have the same angle.
+ */
+class Acrobat
+{
+ public:
+   /*
+    * Implementation of Acrobat State
+    * Each State is a tuple vector.
+    * (theta1, thetha2, angular velocity 1, angular velocity 2)
+    */    
+  class State
+  {
+   public :
+    /**
+     * Construct a state instance.
+     */
+    State(): data(dimension)
+    { /* nothing to do here */ }
+    /**
+     * Construct a state instance from given data.
+     *
+     * @param data Data for the theta and angular velocity of two links.
+     */
+    State(const arma::colvec& data): data(data)
+    { /* nothing to do here */ }
+
+    //! Modify the state representation
+    arma::colvec& Data() {return data;}
+
+    //! Get value of theta1
+    double Theta1() const {return data[0];}
+    //! Modify value of theta1
+    double& Theta1() {return data[0];}
+
+    //! Get value of theta2
+    double Theta2() const {return data[1];}
+    //! Modify value of theta2
+    double& Theta2() {return data[1];}
+
+    //! Get value of Angular velocity 1
+    double AngularVelocity1() const { return data[2]; }
+    //! Modify the angular velocity 1.
+    double& AngularVelocity1() { return data[2]; }
+
+    //! Get value of Angular velocity 2
+    double AngularVelocity2() const { return data[3]; }
+    //! Modify the angular velocity 2.
+    double& AngularVelocity2() { return data[3]; }
+
+    //! Encode the state to a column vector.
+    const arma::colvec& Encode() const { return data; }
+
+    //! Dimension of the encoded state.
+    static constexpr size_t dimension = 4;
+
+   private :
+    //! Locally-Stored (theta1, theta2, angular velocity 1, angular velocity2)
+    arma::colvec data;
+  };
+  /* 
+   * Implementation of action for Acrobat
+   */
+  enum Action
+  {
+    negativeTorque,
+    zeroTorque,
+    positiveTorque,
+    // Track the size of the action space.
+    size
+  };
+
+   /**
+    * Construct a Acrobat instance using the given constants.
+    *
+    * @param gravity gravity
+    * @param linkLength1 length of link 1.
+    * @param linkLength2 length of link 2.
+    * @param linkMass1 mass of link 1.
+    * @param linkMass2 mass of link 2.
+    * @param linkCom1 position of the center of mass of link 1.
+    * @param linkCom2 position of the center of mass of link 2.
+    * @param linkMoi moments of inertia for both link.
+    * @param maxVel1 max angular velocity of link1.
+    * @param maxVel2 max angular velocity of link2.
+    */
+  Acrobat(const double gravity = 9.81,
+          const double linkLength1 = 1.0,
+          const double linkLength2 = 1.0,
+          const double linkMass1 = 1.0,
+          const double linkMass2 = 1.0,
+          const double linkCom1 = 0.5,
+          const double linkCom2 = 0.5,
+          const double linkMoi = 1.0,
+          const double maxVel1 = 4*M_PI,
+          const double maxVel2 = 9*M_PI,
+          const double dt = 0.2) :
+      gravity(gravity),
+      linkLength1(linkLength1),
+      linkLength2(linkLength2),
+      linkMass1(linkMass1),
+      linkMass2(linkMass2),
+      linkCom1(linkCom1),
+      linkCom2(linkCom2),
+      linkMoi(linkMoi),
+      maxVel1(maxVel1),
+      maxVel2(maxVel2),
+      dt(dt)
+  { /* Nothing to do here */ }
+  /**
+   * Dynamics of the Acrobat System.
+   * To get reward and next state based on current
+   * state and current action .
+   * Always return -1 reward
+   *
+   * @param state The current State
+   * @param action The action taken
+   * @param nextState The next state
+   */
+  double Sample(const State& state,
+                const Action& action,
+                State& nextState) const
+  {
+    //! Make a vector to estimate nextstate.
+    arma::colvec state_ = {state.Theta1(), state.Theta2(),
+                          state.AngularVelocity1(),
+                          state.AngularVelocity2()};
+    double torque = Torque(action);
+    arma::colvec nextstate = Rk4(state_, torque);
+
+    nextState.Theta1() = Wrap(nextstate[0], -M_PI, M_PI);
+
+    nextState.Theta2() = Wrap(nextstate[1], -M_PI, M_PI);
+    //! The value of angular velocity is bounded in min and max value.
+    nextState.AngularVelocity1() = std::min(std::max(nextstate[2], -maxVel1),
+                                  maxVel1);
+    nextState.AngularVelocity2() = std::min(std::max(nextstate[3], -maxVel2),
+                                  maxVel2);
+    return -1;
+  };
+  /**
+   * Dynamics of the Acrobat System.
+   * To get reward and next state based on current
+   * state and current action .
+   * This function calls the Sample function to 
+   * estimate the next state return reward
+   * for taking a particular action. 
+   *
+   * @param state The current State
+   * @param action The action taken
+   * @param nextState The next state
+   */
+  double Sample(const State& state, const Action& action) const
+  {
+    State nextState;
+    return Sample(state, action, nextState);
+  }
+  /**
+   * This function does random initialization of 
+   * state space.
+   *
+   */
+  State InitialSample() const
+  {
+    return State((arma::randu<arma::colvec>(4) - 0.5) / 5.0);
+  }
+  /**
+   * This function checks if the acrobat has reached the 
+   * terminal state.
+   * 
+   * @param state The current State
+   */        
+  bool IsTerminal(const State& state) const
+  {
+    return bool (-cos(state.Theta1())-cos(state.Theta1() +
+        state.Theta2()) > 1.0);
+  }
+  /**
+   * This is the ordinary differential equations required for
+   * estimation of nextState through RK4 method.
+   *  
+   * @param state Current State
+   * @param torque Torque Applied 
+   */
+  arma::colvec Dsdt(arma::colvec state,
+                    const double torque) const
+  {
+    double m1 = linkMass1;
+    double m2 = linkMass2;
+    double l1 = linkLength1;
+    double lc1 = linkCom1;
+    double lc2 = linkCom2;
+    double I1 = linkMoi;
+    double I2 = linkMoi;
+    double g = gravity;
+    double a = torque;
+    arma::colvec values(4);
+    double theta1 = state[0];
+    double theta2 = state[1];
+    values[0] = state[2];
+    values[1] = state[3];
+
+    double d1 = m1 * pow(lc1, 2) + m2 *
+               (pow(l1, 2) + pow(lc2, 2) + 2 * l1 * lc2 * cos(theta2))
+               + I1 + I2;
+    double d2 = m2 * (pow(lc2, 2) + l1 * lc2 * cos(theta2)) + I2;
+
+    double phi2 = m2 * lc2 * g * cos(theta1 + theta2 - M_PI / 2.);
+
+    double phi1 = - m2 * l1 * lc2 * pow(values[1], 2) * sin(theta2)
+                 - 2 * m2 * l1 * lc2 * values[1] * values[0] *
+                 sin(theta2) + (m1 * lc1 +  m2 * l1) * g *
+                 cos(theta1 - M_PI / 2) + phi2;
+
+    values[3] = (a + d2 / d1 * phi1 - m2 * l1 * lc2 * pow(values[0], 2) *
+               sin(theta2) - phi2) / (m2 * pow(lc2, 2) + I2 - pow(d2, 2) / d1);
+
+    values[2] = -(d2 * values[3] + phi1) / d1;
+    return values;
+  };
+  /**
+   * Wrap funtion is required to truncate the angle value
+   * from - 180 to 180 .
+   *
+   * This function will make sure that value will always
+   * be between minimum to maximum
+   *
+   * @param value scalar value to wrap
+   * @param minimum minimum range of wrap
+   * @param maximum maximum range of wrap
+   */      
+  double Wrap(double value,
+              double minimum,
+              double maximum) const
+  {
+    double diff = maximum - minimum;
+    if (value > maximum) value = value - diff;
+    else if (value < minimum) value = value + diff;
+    return value;
+  };
+  /**
+   * This function calculates the torque for a
+   * particular action.
+   *
+   * @param Action action taken
+   *
+   * 0 : negative torque
+   * 1 : zero torque
+   * 2 : positive torque
+   */
+  double Torque(const Action& action) const
+  {
+    // Add noise to the Torque
+    /**
+     * Torque is action number - 1.
+     * {0,1,2} -> {-1,0,1} 
+     */
+    return double(action - 1) + mlpack::math::Random(-0.1, 0.1);
+  }
+
+  /**
+   * 
+   * This function calls the RK4 iterative method to estimate the next state
+   * based on given ordinary differential equation.
+   *
+   * The ODE is defined as Dsdt() method in the program.
+   * 
+   * @param state Current State
+   * @param torque Torque applied
+   */
+  arma::colvec Rk4(
+           const arma::colvec state,
+           const double torque) const
+  {
+    arma::colvec k1 = Dsdt(state, torque);
+    arma::colvec k2 = Dsdt(state + dt*k1/2, torque);
+    arma::colvec k3 = Dsdt(state + dt*k2/2, torque);
+    arma::colvec k4 = Dsdt(state + dt*k3, torque);
+    arma::colvec nextState = state + dt*(k1 + 2*k2 + 2*k3 + k4)/6;
+    return nextState;
+  };
+ private:
+  //! Locally-stored gravity.
+  double gravity;
+
+  //! Locally-stored length of link 1.
+  double linkLength1;
+
+  //! Locally-stored length of link 2.
+  double linkLength2;
+
+  //! Locally-stored mass of link 1.
+  double linkMass1;
+
+  //! Locally-stored mass of link 2.
+  double linkMass2;
+
+  //! Locally-stored position of link 1.
+  double linkCom1;
+
+  //! Locally-stored position of link 2.
+  double linkCom2;
+
+  //! Locally-stored moment of intertia value.
+  double linkMoi;
+
+  //! Locally-stored max angular velocity of link1.
+  double maxVel1;
+
+  //! Locally-stored max angular velocity of link2.
+  double maxVel2;
+
+  //! Locally-stored dt for RK4 method
+  double dt;
+}; // class Acrobat
+} // namespace rl
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp
index 600288c966..490b565bf2 100644
--- a/src/mlpack/tests/q_learning_test.cpp
+++ b/src/mlpack/tests/q_learning_test.cpp
@@ -1,6 +1,7 @@
 /**
  * @file q_learning_test.hpp
  * @author Shangtong Zhang
+ * @author Rohan Raj
  *
  * Test for Q-Learning implementation
  *
@@ -17,6 +18,7 @@
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/reinforcement_learning/q_learning.hpp>
 #include <mlpack/methods/reinforcement_learning/environment/mountain_car.hpp>
+#include <mlpack/methods/reinforcement_learning/environment/acrobat.hpp>
 #include <mlpack/methods/reinforcement_learning/environment/cart_pole.hpp>
 #include <mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp>
 #include <mlpack/core/optimizers/adam/adam_update.hpp>
@@ -170,4 +172,68 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDoubleDQN)
   BOOST_REQUIRE(converged);
 }
 
+//! Test DQN in Acrobat task.
+BOOST_AUTO_TEST_CASE(AcrobatWithDQN)
+{
+    // Set up the network.
+  FFN<MeanSquaredError<>, GaussianInitialization> model(MeanSquaredError<>(),
+      GaussianInitialization(0, 0.001));
+  model.Add<Linear<>>(4, 64);
+  model.Add<ReLULayer<>>();
+  model.Add<Linear<>>(64, 32);
+  model.Add<ReLULayer<>>();
+  model.Add<Linear<>>(32, 3);
+
+  // Set up the policy and replay method.
+  GreedyPolicy<Acrobat> policy(1.0, 1000, 0.1);
+  RandomReplay<Acrobat> replayMethod(20, 10000);
+
+  TrainingConfig config;
+  config.StepSize() = 0.01;
+  config.Discount() = 0.99;
+  config.TargetNetworkSyncInterval() = 100;
+  config.ExplorationSteps() = 100;
+  config.DoubleQLearning() = false;
+  config.StepLimit() = 400;
+
+  // Set up DQN agent.
+  QLearning<Acrobat, decltype(model), AdamUpdate, decltype(policy)>
+      agent(std::move(config), std::move(model), std::move(policy),
+          std::move(replayMethod));
+
+  arma::running_stat<double> averageReturn;
+  size_t episodes = 0;
+  bool converged = true;
+  while (true)
+  {
+    double episodeReturn = agent.Episode();
+    averageReturn(episodeReturn);
+    episodes += 1;
+
+    if (episodes > 1000)
+    {
+      Log::Debug << "Acrobat with DQN failed." << std::endl;
+      converged = false;
+      break;
+    }
+
+    /**
+     * I am using a thresold of -380 to check convegence.
+     */
+    Log::Debug << "Average return: " << averageReturn.mean()
+        << " Episode return: " << episodeReturn << std::endl;
+    if (averageReturn.mean() > -380.00)
+    {
+      agent.Deterministic() = true;
+      arma::running_stat<double> testReturn;
+      for (size_t i = 0; i < 20; ++i)
+        testReturn(agent.Episode());
+
+      Log::Debug << "Average return in deterministic test: "
+          << testReturn.mean() << std::endl;
+      break;
+    }
+  }
+  BOOST_REQUIRE(converged);
+}
 BOOST_AUTO_TEST_SUITE_END();
diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp
index 5ee77f3350..2d7026ac7c 100644
--- a/src/mlpack/tests/rl_components_test.cpp
+++ b/src/mlpack/tests/rl_components_test.cpp
@@ -14,6 +14,7 @@
 
 #include <mlpack/methods/reinforcement_learning/environment/mountain_car.hpp>
 #include <mlpack/methods/reinforcement_learning/environment/cart_pole.hpp>
+#include <mlpack/methods/reinforcement_learning/environment/acrobat.hpp>
 #include <mlpack/methods/reinforcement_learning/replay/random_replay.hpp>
 #include <mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp>
 
@@ -25,6 +26,23 @@ using namespace mlpack::rl;
 
 BOOST_AUTO_TEST_SUITE(RLComponentsTest)
 
+/**
+ * Constructs a Acrobat instance and check if the main rountine works as
+ * it should be.
+ */
+BOOST_AUTO_TEST_CASE(SimpleAcrobatTest)
+{
+  const Acrobat task = Acrobat();
+
+  Acrobat::State state = task.InitialSample();
+  Acrobat::Action action = Acrobat::Action::negativeTorque;
+  double reward = task.Sample(state, action);
+
+  BOOST_REQUIRE_EQUAL(reward, -1.0);
+  BOOST_REQUIRE(!task.IsTerminal(state));
+  BOOST_REQUIRE_EQUAL(3, Acrobat::Action::size);
+}
+
 /**
  * Constructs a MountainCar instance and check if the main rountine works as
  * it should be.

From 6df8d6366166dec2aa42434bf7f91e87a3e41191 Mon Sep 17 00:00:00 2001
From: Marcus Edel <marcus.edel@fu-berlin.de>
Date: Mon, 16 Apr 2018 21:18:35 +0200
Subject: [PATCH 63/67] Minor style fixes (punctuation, comments).

---
 .../environment/acrobat.hpp                   | 299 +++++++++---------
 src/mlpack/tests/q_learning_test.cpp          |   8 +-
 2 files changed, 155 insertions(+), 152 deletions(-)

diff --git a/src/mlpack/methods/reinforcement_learning/environment/acrobat.hpp b/src/mlpack/methods/reinforcement_learning/environment/acrobat.hpp
index bacd9a208f..21816f87bd 100644
--- a/src/mlpack/methods/reinforcement_learning/environment/acrobat.hpp
+++ b/src/mlpack/methods/reinforcement_learning/environment/acrobat.hpp
@@ -17,59 +17,58 @@
 
 namespace mlpack{
 namespace rl{
+
 /**
- * Implementation of Acrobat game
- * Acrobot is a 2-link pendulum with only the second joint actuated
- * Intitially, both links point downwards. The goal is to swing the
- * end-effector at a height at least the length of one link above the base.
- * Both links can swing freely and can pass by each other, i.e., they don't
- * collide when they have the same angle.
+ * Implementation of Acrobat game. Acrobot is a 2-link pendulum with only the
+ * second joint actuated. Intitially, both links point downwards. The goal is
+ * to swing the end-effector at a height at least the length of one link above
+ * the base. Both links can swing freely and can pass by each other, i.e.,
+ * they don't collide when they have the same angle.
  */
 class Acrobat
 {
  public:
    /*
-    * Implementation of Acrobat State
-    * Each State is a tuple vector.
-    * (theta1, thetha2, angular velocity 1, angular velocity 2)
-    */    
+    * Implementation of Acrobat State. Each State is a tuple vector
+    * (theta1, thetha2, angular velocity 1, angular velocity 2).
+    */
   class State
   {
-   public :
+   public:
     /**
      * Construct a state instance.
      */
-    State(): data(dimension)
-    { /* nothing to do here */ }
+    State(): data(dimension) { /* nothing to do here */ }
+
     /**
      * Construct a state instance from given data.
      *
      * @param data Data for the theta and angular velocity of two links.
      */
-    State(const arma::colvec& data): data(data)
+    State(const arma::colvec& data) : data(data)
     { /* nothing to do here */ }
 
-    //! Modify the state representation
-    arma::colvec& Data() {return data;}
+    //! Modify the state representation.
+    arma::colvec& Data() { return data; }
 
-    //! Get value of theta1
-    double Theta1() const {return data[0];}
-    //! Modify value of theta1
-    double& Theta1() {return data[0];}
+    //! Get value of theta (one).
+    double Theta1() const { return data[0]; }
+    //! Modify value of theta (one).
+    double& Theta1() { return data[0]; }
 
-    //! Get value of theta2
-    double Theta2() const {return data[1];}
-    //! Modify value of theta2
-    double& Theta2() {return data[1];}
+    //! Get value of theta (two).
+    double Theta2() const { return data[1]; }
+    //! Modify value of theta (two).
+    double& Theta2() { return data[1]; }
 
-    //! Get value of Angular velocity 1
+    //! Get value of Angular velocity (one).
     double AngularVelocity1() const { return data[2]; }
-    //! Modify the angular velocity 1.
+    //! Modify the angular velocity (one).
     double& AngularVelocity1() { return data[2]; }
 
-    //! Get value of Angular velocity 2
+    //! Get value of Angular velocity (two).
     double AngularVelocity2() const { return data[3]; }
-    //! Modify the angular velocity 2.
+    //! Modify the angular velocity (two).
     double& AngularVelocity2() { return data[3]; }
 
     //! Encode the state to a column vector.
@@ -78,11 +77,12 @@ class Acrobat
     //! Dimension of the encoded state.
     static constexpr size_t dimension = 4;
 
-   private :
-    //! Locally-Stored (theta1, theta2, angular velocity 1, angular velocity2)
+   private:
+    //! Locally-Stored (theta1, theta2, angular velocity 1, angular velocity2).
     arma::colvec data;
   };
-  /* 
+
+  /*
    * Implementation of action for Acrobat
    */
   enum Action
@@ -90,6 +90,7 @@ class Acrobat
     negativeTorque,
     zeroTorque,
     positiveTorque,
+
     // Track the size of the action space.
     size
   };
@@ -97,16 +98,17 @@ class Acrobat
    /**
     * Construct a Acrobat instance using the given constants.
     *
-    * @param gravity gravity
-    * @param linkLength1 length of link 1.
-    * @param linkLength2 length of link 2.
-    * @param linkMass1 mass of link 1.
-    * @param linkMass2 mass of link 2.
-    * @param linkCom1 position of the center of mass of link 1.
-    * @param linkCom2 position of the center of mass of link 2.
-    * @param linkMoi moments of inertia for both link.
-    * @param maxVel1 max angular velocity of link1.
-    * @param maxVel2 max angular velocity of link2.
+    * @param gravity The gravity parameter.
+    * @param linkLength1 The length of link 1.
+    * @param linkLength2 The length of link 2.
+    * @param linkMass1 The mass of link 1.
+    * @param linkMass2 The mass of link 2.
+    * @param linkCom1 The position of the center of mass of link 1.
+    * @param linkCom2 The position of the center of mass of link 2.
+    * @param linkMoi The moments of inertia for both link.
+    * @param maxVel1 The max angular velocity of link1.
+    * @param maxVel2 The max angular velocity of link2.
+    * @param dt The differential value.
     */
   Acrobat(const double gravity = 9.81,
           const double linkLength1 = 1.0,
@@ -116,8 +118,8 @@ class Acrobat
           const double linkCom1 = 0.5,
           const double linkCom2 = 0.5,
           const double linkMoi = 1.0,
-          const double maxVel1 = 4*M_PI,
-          const double maxVel2 = 9*M_PI,
+          const double maxVel1 = 4 * M_PI,
+          const double maxVel2 = 9 * M_PI,
           const double dt = 0.2) :
       gravity(gravity),
       linkLength1(linkLength1),
@@ -131,178 +133,176 @@ class Acrobat
       maxVel2(maxVel2),
       dt(dt)
   { /* Nothing to do here */ }
+
   /**
-   * Dynamics of the Acrobat System.
-   * To get reward and next state based on current
-   * state and current action .
-   * Always return -1 reward
+   * Dynamics of the Acrobat System. To get reward and next state based on
+   * current state and current action. Always return -1 reward.
    *
-   * @param state The current State
-   * @param action The action taken
-   * @param nextState The next state
+   * @param state The current State.
+   * @param action The action taken.
+   * @param nextState The next state.
+   * @return reward, it's always -1.0.
    */
   double Sample(const State& state,
                 const Action& action,
                 State& nextState) const
   {
-    //! Make a vector to estimate nextstate.
-    arma::colvec state_ = {state.Theta1(), state.Theta2(),
-                          state.AngularVelocity1(),
-                          state.AngularVelocity2()};
-    double torque = Torque(action);
-    arma::colvec nextstate = Rk4(state_, torque);
+    // Make a vector to estimate nextstate.
+    arma::colvec currentState = {state.Theta1(), state.Theta2(),
+        state.AngularVelocity1(), state.AngularVelocity2()};
 
-    nextState.Theta1() = Wrap(nextstate[0], -M_PI, M_PI);
+    arma::colvec currentNextState = Rk4(currentState, Torque(action));
 
-    nextState.Theta2() = Wrap(nextstate[1], -M_PI, M_PI);
+    nextState.Theta1() = Wrap(currentNextState[0], -M_PI, M_PI);
+
+    nextState.Theta2() = Wrap(currentNextState[1], -M_PI, M_PI);
     //! The value of angular velocity is bounded in min and max value.
-    nextState.AngularVelocity1() = std::min(std::max(nextstate[2], -maxVel1),
-                                  maxVel1);
-    nextState.AngularVelocity2() = std::min(std::max(nextstate[3], -maxVel2),
-                                  maxVel2);
-    return -1;
+    nextState.AngularVelocity1() = std::min(
+        std::max(currentNextState[2], -maxVel1), maxVel1);
+    nextState.AngularVelocity2() = std::min(
+        std::max(currentNextState[3], -maxVel2), maxVel2);
+
+    return -1.0;
   };
+
   /**
-   * Dynamics of the Acrobat System.
-   * To get reward and next state based on current
-   * state and current action .
-   * This function calls the Sample function to 
-   * estimate the next state return reward
-   * for taking a particular action. 
+   * Dynamics of the Acrobat System. To get reward and next state based on
+   * current state and current action. This function calls the Sample function
+   * to estimate the next state return reward for taking a particular action.
    *
-   * @param state The current State
-   * @param action The action taken
-   * @param nextState The next state
+   * @param state The current State.
+   * @param action The action taken.
+   * @param nextState The next state.
    */
   double Sample(const State& state, const Action& action) const
   {
     State nextState;
     return Sample(state, action, nextState);
   }
+
   /**
-   * This function does random initialization of 
-   * state space.
-   *
+   * This function does random initialization of state space.
    */
   State InitialSample() const
   {
     return State((arma::randu<arma::colvec>(4) - 0.5) / 5.0);
   }
+
   /**
-   * This function checks if the acrobat has reached the 
-   * terminal state.
-   * 
-   * @param state The current State
-   */        
+   * This function checks if the acrobat has reached the terminal state.
+   *
+   * @param state The current State.
+   */
   bool IsTerminal(const State& state) const
   {
-    return bool (-cos(state.Theta1())-cos(state.Theta1() +
+    return bool (-std::cos(state.Theta1())-std::cos(state.Theta1() +
         state.Theta2()) > 1.0);
   }
+
   /**
-   * This is the ordinary differential equations required for
-   * estimation of nextState through RK4 method.
-   *  
-   * @param state Current State
-   * @param torque Torque Applied 
+   * This is the ordinary differential equations required for estimation of
+   * nextState through RK4 method.
+   *
+   * @param state Current State.
+   * @param torque The torque Applied.
    */
-  arma::colvec Dsdt(arma::colvec state,
-                    const double torque) const
+  arma::colvec Dsdt(arma::colvec state, const double torque) const
   {
-    double m1 = linkMass1;
-    double m2 = linkMass2;
-    double l1 = linkLength1;
-    double lc1 = linkCom1;
-    double lc2 = linkCom2;
-    double I1 = linkMoi;
-    double I2 = linkMoi;
-    double g = gravity;
-    double a = torque;
+    const double m1 = linkMass1;
+    const double m2 = linkMass2;
+    const double l1 = linkLength1;
+    const double lc1 = linkCom1;
+    const double lc2 = linkCom2;
+    const double I1 = linkMoi;
+    const double I2 = linkMoi;
+    const double g = gravity;
+    const double a = torque;
+    const double theta1 = state[0];
+    const double theta2 = state[1];
+
     arma::colvec values(4);
-    double theta1 = state[0];
-    double theta2 = state[1];
     values[0] = state[2];
     values[1] = state[3];
 
-    double d1 = m1 * pow(lc1, 2) + m2 *
-               (pow(l1, 2) + pow(lc2, 2) + 2 * l1 * lc2 * cos(theta2))
-               + I1 + I2;
-    double d2 = m2 * (pow(lc2, 2) + l1 * lc2 * cos(theta2)) + I2;
+    const double d1 = m1 * std::pow(lc1, 2) + m2 * (std::pow(l1, 2) +
+        std::pow(lc2, 2) + 2 * l1 * lc2 * std::cos(theta2)) + I1 + I2;
 
-    double phi2 = m2 * lc2 * g * cos(theta1 + theta2 - M_PI / 2.);
+    const double d2 = m2 * (std::pow(lc2, 2) + l1 * lc2 * std::cos(theta2)) + I2;
 
-    double phi1 = - m2 * l1 * lc2 * pow(values[1], 2) * sin(theta2)
-                 - 2 * m2 * l1 * lc2 * values[1] * values[0] *
-                 sin(theta2) + (m1 * lc1 +  m2 * l1) * g *
-                 cos(theta1 - M_PI / 2) + phi2;
+    const double phi2 = m2 * lc2 * g * std::cos(theta1 + theta2 - M_PI / 2.);
 
-    values[3] = (a + d2 / d1 * phi1 - m2 * l1 * lc2 * pow(values[0], 2) *
-               sin(theta2) - phi2) / (m2 * pow(lc2, 2) + I2 - pow(d2, 2) / d1);
+    const double phi1 = - m2 * l1 * lc2 * std::pow(values[1], 2) *
+        std::sin(theta2) - 2 * m2 * l1 * lc2 * values[1] * values[0] *
+        std::sin(theta2) + (m1 * lc1 +  m2 * l1) * g *
+        std::cos(theta1 - M_PI / 2) + phi2;
+
+    values[3] = (a + d2 / d1 * phi1 - m2 * l1 * lc2 * std::pow(values[0], 2) *
+        std::sin(theta2) - phi2) / (m2 * std::pow(lc2, 2) + I2 -
+        std::pow(d2, 2) / d1);
 
     values[2] = -(d2 * values[3] + phi1) / d1;
+
     return values;
   };
+
   /**
-   * Wrap funtion is required to truncate the angle value
-   * from - 180 to 180 .
+   * Wrap funtion is required to truncate the angle value from -180 to 180.
+   * This function will make sure that value will always be between minimum
+   * to maximum.
    *
-   * This function will make sure that value will always
-   * be between minimum to maximum
-   *
-   * @param value scalar value to wrap
-   * @param minimum minimum range of wrap
-   * @param maximum maximum range of wrap
-   */      
+   * @param value Scalar value to wrap.
+   * @param minimum Minimum range of wrap.
+   * @param maximum Maximum range of wrap.
+   */
   double Wrap(double value,
-              double minimum,
-              double maximum) const
+              const double minimum,
+              const double maximum) const
   {
-    double diff = maximum - minimum;
-    if (value > maximum) value = value - diff;
-    else if (value < minimum) value = value + diff;
+    const double diff = maximum - minimum;
+
+    if (value > maximum)
+    {
+      value = value - diff;
+    }
+    else if (value < minimum)
+    {
+      value = value + diff;
+    }
+
     return value;
   };
+
   /**
-   * This function calculates the torque for a
-   * particular action.
+   * This function calculates the torque for a particular action.
+   * 0 : negative torque, 1 : zero torque, 2 : positive torque.
    *
-   * @param Action action taken
-   *
-   * 0 : negative torque
-   * 1 : zero torque
-   * 2 : positive torque
+   * @param Action action taken.
    */
   double Torque(const Action& action) const
   {
-    // Add noise to the Torque
-    /**
-     * Torque is action number - 1.
-     * {0,1,2} -> {-1,0,1} 
-     */
+    // Add noise to the Torque Torque is action number - 1. {0,1,2} -> {-1,0,1}.
     return double(action - 1) + mlpack::math::Random(-0.1, 0.1);
   }
 
   /**
-   * 
+   *
    * This function calls the RK4 iterative method to estimate the next state
    * based on given ordinary differential equation.
    *
-   * The ODE is defined as Dsdt() method in the program.
-   * 
-   * @param state Current State
-   * @param torque Torque applied
+   * @param state The current State.
+   * @param torque The torque applied.
    */
-  arma::colvec Rk4(
-           const arma::colvec state,
-           const double torque) const
+  arma::colvec Rk4(const arma::colvec state, const double torque) const
   {
     arma::colvec k1 = Dsdt(state, torque);
-    arma::colvec k2 = Dsdt(state + dt*k1/2, torque);
-    arma::colvec k3 = Dsdt(state + dt*k2/2, torque);
-    arma::colvec k4 = Dsdt(state + dt*k3, torque);
-    arma::colvec nextState = state + dt*(k1 + 2*k2 + 2*k3 + k4)/6;
+    arma::colvec k2 = Dsdt(state + dt * k1 / 2, torque);
+    arma::colvec k3 = Dsdt(state + dt * k2 / 2, torque);
+    arma::colvec k4 = Dsdt(state + dt * k3, torque);
+    arma::colvec nextState = state + dt * (k1 + 2 * k2 + 2 * k3 + k4) / 6;
+
     return nextState;
   };
+
  private:
   //! Locally-stored gravity.
   double gravity;
@@ -334,9 +334,10 @@ class Acrobat
   //! Locally-stored max angular velocity of link2.
   double maxVel2;
 
-  //! Locally-stored dt for RK4 method
+  //! Locally-stored dt for RK4 method.
   double dt;
 }; // class Acrobat
+
 } // namespace rl
 } // namespace mlpack
 
diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp
index 490b565bf2..6e7613d70b 100644
--- a/src/mlpack/tests/q_learning_test.cpp
+++ b/src/mlpack/tests/q_learning_test.cpp
@@ -62,7 +62,7 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDQN)
   // Set up DQN agent.
   QLearning<CartPole, decltype(model), AdamUpdate, decltype(policy)>
       agent(std::move(config), std::move(model), std::move(policy),
-          std::move(replayMethod));
+      std::move(replayMethod));
 
   arma::running_stat<double> averageReturn;
   size_t episodes = 0;
@@ -135,7 +135,7 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDoubleDQN)
     // Set up the DQN agent.
     QLearning<CartPole, decltype(model), RMSPropUpdate, decltype(policy)>
         agent(std::move(config), std::move(model), std::move(policy),
-            std::move(replayMethod));
+        std::move(replayMethod));
 
     arma::running_stat<double> averageReturn;
 
@@ -199,7 +199,7 @@ BOOST_AUTO_TEST_CASE(AcrobatWithDQN)
   // Set up DQN agent.
   QLearning<Acrobat, decltype(model), AdamUpdate, decltype(policy)>
       agent(std::move(config), std::move(model), std::move(policy),
-          std::move(replayMethod));
+      std::move(replayMethod));
 
   arma::running_stat<double> averageReturn;
   size_t episodes = 0;
@@ -234,6 +234,8 @@ BOOST_AUTO_TEST_CASE(AcrobatWithDQN)
       break;
     }
   }
+
   BOOST_REQUIRE(converged);
 }
+
 BOOST_AUTO_TEST_SUITE_END();

From 789c90247d41c57c543864648169c31540f27daa Mon Sep 17 00:00:00 2001
From: luffy <rajrohan1108@gmail.com>
Date: Tue, 17 Apr 2018 09:51:37 +0530
Subject: [PATCH 64/67] Minor Style

---
 .../methods/reinforcement_learning/environment/acrobat.hpp     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/mlpack/methods/reinforcement_learning/environment/acrobat.hpp b/src/mlpack/methods/reinforcement_learning/environment/acrobat.hpp
index 21816f87bd..713121ae80 100644
--- a/src/mlpack/methods/reinforcement_learning/environment/acrobat.hpp
+++ b/src/mlpack/methods/reinforcement_learning/environment/acrobat.hpp
@@ -227,7 +227,8 @@ class Acrobat
     const double d1 = m1 * std::pow(lc1, 2) + m2 * (std::pow(l1, 2) +
         std::pow(lc2, 2) + 2 * l1 * lc2 * std::cos(theta2)) + I1 + I2;
 
-    const double d2 = m2 * (std::pow(lc2, 2) + l1 * lc2 * std::cos(theta2)) + I2;
+    const double d2 = m2 * (std::pow(lc2, 2) + l1 * lc2 * std::cos(theta2)) +
+        I2;
 
     const double phi2 = m2 * lc2 * g * std::cos(theta1 + theta2 - M_PI / 2.);
 

From 114137eec8c025d62e77e2c9379ce5f0f76943b9 Mon Sep 17 00:00:00 2001
From: Marcus Edel <marcus.edel@fu-berlin.de>
Date: Wed, 18 Apr 2018 00:39:38 +0200
Subject: [PATCH 65/67] Minor style fixes (comment, citation).

---
 src/mlpack/methods/ann/init_rules/he_init.hpp | 12 ++++-----
 .../ann/init_rules/lecun_normal_init.hpp      | 27 ++++++++++---------
 src/mlpack/tests/init_rules_test.cpp          |  8 +++---
 3 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp
index 822fb5a81e..421b499dc3 100644
--- a/src/mlpack/methods/ann/init_rules/he_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/he_init.hpp
@@ -31,13 +31,14 @@ namespace ann /** Artificial Neural Network. */ {
  * For more information, the following paper can be referred to:
  *
  * @code
- * @article{He2015DelvingDI,
+ * @article{Delving2015,
  *   title   = {Delving Deep into Rectifiers: Surpassing Human-Level Performance
  *              on ImageNet Classification},
  *   author  = {Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun},
  *   journal = {2015 IEEE International Conference on Computer Vision (ICCV)},
  *   year    = {2015},
- *   pages   = {1026-1034}}
+ *   pages   = {1026-1034}
+ * }
  * @endcode
  *
  */
@@ -46,7 +47,6 @@ class HeInitialization
  public:
   /**
    * Initialize the HeInitialization object.
-   *
    */
   HeInitialization()
   {
@@ -61,9 +61,7 @@ class HeInitialization
    * @param rows Number of rows.
    * @param cols Number of columns.
    */
-  void Initialize(arma::mat& W,
-                  const size_t rows,
-                  const size_t cols)
+  void Initialize(arma::mat& W, const size_t rows, const size_t cols)
   {
     // He initialization rule says to initialize weights with random
     // values taken from a gaussian distribution with mean = 0 and
@@ -76,7 +74,7 @@ class HeInitialization
     }
 
     // Multipling a random variable X with variance V(X) by some factor c,
-    // then the variance V(cX) = (c^2)* V(X).
+    // then the variance V(cX) = (c^2) * V(X).
     W.imbue( [&]() { return sqrt(variance) * arma::randn(); } );
   }
 
diff --git a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
index 41ad1e9121..19bfb46173 100644
--- a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp
@@ -28,19 +28,21 @@ namespace ann /** Artificial Neural Network. */ {
  * For more information, the following papers can be referred to:
  *
  * @code
- * @inproceedings{conf/nips/KlambauerUMH17,
- * title  = {Self-Normalizing Neural Networks.},
- * author = {Klambauer, Günter and Unterthiner, Thomas
- *           and Mayr, Andreas and Hochreiter, Sepp},
- * pages  = {972-981},
- * year   = 2017}
+ * @inproceedings{Klambauer2017,
+ *   itle  = {Self-Normalizing Neural Networks.},
+ *   author = {Klambauer, Günter and Unterthiner, Thomas
+ *             and Mayr, Andreas and Hochreiter, Sepp},
+ *   pages  = {972-981},
+ *   year   = {2017}
+ * }
  *
- * @inproceedings{LeCun:1998:EB:645754.668382,
- * title  = {Efficient BackProp},
- * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B.
- *          and M\"{u}ller, Klaus-Robert},
- * year   = {1998},
- * pages  = {9--50}}
+ * @inproceedings{LeCun1998,
+ *   title  = {Efficient BackProp},
+ *   author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B.
+ *             and M\"{u}ller, Klaus-Robert},
+ *   year   = {1998},
+ *   pages  = {9--50}
+ * }
  * @endcode
  *
 */
@@ -49,7 +51,6 @@ class LecunNormalInitialization
  public:
   /**
    * Initialize the LecunNormalInitialization object.
-   *
    */
   LecunNormalInitialization()
   {
diff --git a/src/mlpack/tests/init_rules_test.cpp b/src/mlpack/tests/init_rules_test.cpp
index 5d2677b26a..692bf57302 100644
--- a/src/mlpack/tests/init_rules_test.cpp
+++ b/src/mlpack/tests/init_rules_test.cpp
@@ -331,8 +331,8 @@ BOOST_AUTO_TEST_CASE(GlorotInitNormalTest)
 }
 
 /**
-* Simple test of the HeInitialization class.
-*/
+ * Simple test of the HeInitialization class.
+ */
 BOOST_AUTO_TEST_CASE(HeInitTest)
 {
   const size_t rows = 4;
@@ -356,8 +356,8 @@ BOOST_AUTO_TEST_CASE(HeInitTest)
 }
 
 /**
-* Simple test of the LecunNormalInitialization class.
-*/
+ * Simple test of the LecunNormalInitialization class.
+ */
 BOOST_AUTO_TEST_CASE(LecunNormalInitTest)
 {
   const size_t rows = 4;

From 9c84d9547e4d69a8b49554a0e8b718bdc915c04a Mon Sep 17 00:00:00 2001
From: Conrad Sanderson <conradsnicta@users.noreply.github.com>
Date: Thu, 19 Apr 2018 09:08:30 +0200
Subject: [PATCH 66/67] fix unused result warning

---
 src/mlpack/tests/arma_extend_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/tests/arma_extend_test.cpp b/src/mlpack/tests/arma_extend_test.cpp
index 175528ea7f..0594fb567b 100644
--- a/src/mlpack/tests/arma_extend_test.cpp
+++ b/src/mlpack/tests/arma_extend_test.cpp
@@ -157,7 +157,7 @@ BOOST_AUTO_TEST_CASE(MatRowColIteratorDecrementOperatorTest)
 
   // Check that postfix-- does not decrement the position when position is
   // pointing to the beginning.
-  (void) it2--;
+  auto junk = it2--; (void)(junk);
   BOOST_REQUIRE_EQUAL(it1.row(), it2.row());
   BOOST_REQUIRE_EQUAL(it1.col(), it2.col());
 

From 282ea25b316a9efe1938dcf7f08157308eee5be8 Mon Sep 17 00:00:00 2001
From: Marcus Edel <marcus.edel@fu-berlin.de>
Date: Fri, 20 Apr 2018 22:10:28 +0200
Subject: [PATCH 67/67] Minor style fixes (comments, indentation).

---
 src/mlpack/methods/ann/rnn_impl.hpp         | 10 +++-----
 src/mlpack/tests/recurrent_network_test.cpp | 28 +++++++++++----------
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/mlpack/methods/ann/rnn_impl.hpp b/src/mlpack/methods/ann/rnn_impl.hpp
index bc93b5fa89..8e86ef035c 100644
--- a/src/mlpack/methods/ann/rnn_impl.hpp
+++ b/src/mlpack/methods/ann/rnn_impl.hpp
@@ -290,23 +290,21 @@ void RNN<OutputLayerType, InitializationRuleType, CustomLayers...>::Gradient(
     {
       outputLayer.Backward(std::move(boost::apply_visitor(
           outputParameterVisitor, network.back())),
-              std::move(arma::mat(responses.slice(0).colptr(begin),
-                  responses.n_rows, batchSize, false, true)),
-          std::move(error));
+          std::move(arma::mat(responses.slice(0).colptr(begin),
+          responses.n_rows, batchSize, false, true)), std::move(error));
     }
     else
     {
       outputLayer.Backward(std::move(boost::apply_visitor(
           outputParameterVisitor, network.back())),
           std::move(arma::mat(responses.slice(rho - seqNum - 1).colptr(begin),
-              responses.n_rows, batchSize, false, true)),
-          std::move(error));
+          responses.n_rows, batchSize, false, true)), std::move(error));
     }
 
     Backward();
     Gradient(std::move(
         arma::mat(predictors.slice(rho - seqNum - 1).colptr(begin),
-            predictors.n_rows, batchSize, false, true)));
+        predictors.n_rows, batchSize, false, true)));
     gradient += currentGradient;
   }
 }
diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp
index 0f8fb5b46e..46e528474e 100644
--- a/src/mlpack/tests/recurrent_network_test.cpp
+++ b/src/mlpack/tests/recurrent_network_test.cpp
@@ -1122,6 +1122,7 @@ void GenerateNoisySinRNN(arma::cube& data,
 {
   int points = dataPoints;
   int r = dataPoints % rho;
+
   if (r == 0)
   {
     points += outputSteps;
@@ -1130,14 +1131,16 @@ void GenerateNoisySinRNN(arma::cube& data,
   {
     points += rho - r + outputSteps;
   }
+
   arma::colvec x(points);
   int i = 0;
   double interval = numCycles / freq / points;
+
   x.for_each([&i, gain, freq, phase, noisePercent, interval]
-             (arma::colvec::elem_type& val) {
+    (arma::colvec::elem_type& val) {
     double t = interval * (i++);
     val = gain * ::sin(2 * M_PI * freq * t + phase) +
-          (noisePercent * gain / 100 * Random(0.0, 0.1));
+        (noisePercent * gain / 100 * Random(0.0, 0.1));
   });
 
   arma::colvec y = x;
@@ -1148,6 +1151,7 @@ void GenerateNoisySinRNN(arma::cube& data,
   size_t numColumns = y.n_elem / rho;
   data = arma::cube(1, numColumns, rho);
   labels = arma::cube(outputSteps, numColumns, 1);
+
   for (size_t i = 0; i < numColumns; ++i)
   {
     data.tube(0, i) = y.rows(i * rho, i * rho + rho - 1);
@@ -1181,20 +1185,16 @@ double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100)
   // Break into training and test sets. Simply split along columns.
   size_t trainCols = data.n_cols * 0.8; // Take 20% out for testing.
   size_t testCols = data.n_cols - trainCols;
-  arma::cube testData =
-      data.subcube(0, data.n_cols - testCols, 0, data.n_rows - 1,
-                   data.n_cols - 1, data.n_slices - 1);
-  arma::cube testLabels =
-      labels.subcube(0, labels.n_cols - testCols, 0, labels.n_rows - 1,
-                     labels.n_cols - 1, labels.n_slices - 1);
+  arma::cube testData = data.subcube(0, data.n_cols - testCols, 0,
+      data.n_rows - 1, data.n_cols - 1, data.n_slices - 1);
+  arma::cube testLabels = labels.subcube(0, labels.n_cols - testCols, 0,
+      labels.n_rows - 1, labels.n_cols - 1, labels.n_slices - 1);
 
   for (size_t i = 0; i < numEpochs; ++i)
   {
     net.Train(data.subcube(0, 0, 0, data.n_rows - 1, trainCols - 1,
-                           data.n_slices - 1),
-              labels.subcube(0, 0, 0, labels.n_rows - 1, trainCols - 1,
-                             labels.n_slices - 1),
-              opt);
+        data.n_slices - 1), labels.subcube(0, 0, 0, labels.n_rows - 1,
+        trainCols - 1, labels.n_slices - 1), opt);
   }
   // Well now it should be trained. Do the test here.
   arma::cube prediction;
@@ -1204,11 +1204,13 @@ double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100)
   // data and the pediction to vectors and compare the two.
   arma::colvec testVector = arma::vectorise(testData);
   arma::colvec predVector = arma::vectorise(prediction);
+
   // Adjust the vectors for comparison, as the prediction is one step ahead.
   testVector = testVector.rows(1, testVector.n_rows - 1);
   predVector = predVector.rows(0, predVector.n_rows - 2);
   double error = std::sqrt(arma::sum(arma::square(testVector - predVector))) /
-                 testVector.n_rows;
+      testVector.n_rows;
+
   return error;
 }