From 5736028967789210e6ea52121685c644f5d61c78 Mon Sep 17 00:00:00 2001
From: Saksham Bansal <7020962+saksham189@users.noreply.github.com>
Date: Wed, 19 Jun 2019 01:18:50 +0700
Subject: [PATCH 01/11] Add Regularizers

---
 src/mlpack/methods/ann/CMakeLists.txt         |  1 +
 src/mlpack/methods/ann/layer/layer_types.hpp  | 11 ++-
 src/mlpack/methods/ann/layer/linear.hpp       | 10 ++-
 src/mlpack/methods/ann/layer/linear_impl.hpp  | 44 +++++++----
 .../methods/ann/regularizer/CMakeLists.txt    | 15 ++++
 .../methods/ann/regularizer/lregularizer.hpp  | 78 +++++++++++++++++++
 .../ann/regularizer/lregularizer_impl.hpp     | 64 +++++++++++++++
 7 files changed, 203 insertions(+), 20 deletions(-)
 create mode 100644 src/mlpack/methods/ann/regularizer/CMakeLists.txt
 create mode 100644 src/mlpack/methods/ann/regularizer/lregularizer.hpp
 create mode 100644 src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp

diff --git a/src/mlpack/methods/ann/CMakeLists.txt b/src/mlpack/methods/ann/CMakeLists.txt
index f6e3c5dab3..8e4118e3ad 100644
--- a/src/mlpack/methods/ann/CMakeLists.txt
+++ b/src/mlpack/methods/ann/CMakeLists.txt
@@ -18,6 +18,7 @@ add_subdirectory(convolution_rules)
 add_subdirectory(gan)
 add_subdirectory(rbm)
 add_subdirectory(augmented)
+add_subdirectory(regularizer)
 
 # Add directory name to sources.
 set(DIR_SRCS)
diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp
index fc6bf6f69a..8bc2fa5aa3 100644
--- a/src/mlpack/methods/ann/layer/layer_types.hpp
+++ b/src/mlpack/methods/ann/layer/layer_types.hpp
@@ -46,6 +46,9 @@
 #include <mlpack/methods/ann/convolution_rules/naive_convolution.hpp>
 #include <mlpack/methods/ann/convolution_rules/fft_convolution.hpp>
 
+// Regularizers.
+#include <mlpack/methods/ann/regularizer/lregularizer.hpp>
+
 // Loss function modules.
 #include <mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp>
 
@@ -56,7 +59,6 @@ template<typename InputDataType, typename OutputDataType> class BatchNorm;
 template<typename InputDataType, typename OutputDataType> class DropConnect;
 template<typename InputDataType, typename OutputDataType> class Glimpse;
 template<typename InputDataType, typename OutputDataType> class LayerNorm;
-template<typename InputDataType, typename OutputDataType> class Linear;
 template<typename InputDataType, typename OutputDataType> class LinearNoBias;
 template<typename InputDataType, typename OutputDataType> class LSTM;
 template<typename InputDataType, typename OutputDataType> class GRU;
@@ -64,6 +66,11 @@ template<typename InputDataType, typename OutputDataType> class FastLSTM;
 template<typename InputDataType, typename OutputDataType> class VRClassReward;
 template<typename InputDataType, typename OutputDataType> class Concatenate;
 
+template<typename InputDataType,
+         typename OutputDataType,
+         typename RegularizerType>
+class Linear;
+
 template<typename InputDataType,
          typename OutputDataType
 >
@@ -177,7 +184,7 @@ using LayerTypes = boost::variant<
     LayerNorm<arma::mat, arma::mat>*,
     LeakyReLU<arma::mat, arma::mat>*,
     CReLU<arma::mat, arma::mat>*,
-    Linear<arma::mat, arma::mat>*,
+    Linear<arma::mat, arma::mat, L2Regularizer>*,
     LinearNoBias<arma::mat, arma::mat>*,
     LogSoftMax<arma::mat, arma::mat>*,
     Lookup<arma::mat, arma::mat>*,
diff --git a/src/mlpack/methods/ann/layer/linear.hpp b/src/mlpack/methods/ann/layer/linear.hpp
index 362a626e87..b73f242208 100644
--- a/src/mlpack/methods/ann/layer/linear.hpp
+++ b/src/mlpack/methods/ann/layer/linear.hpp
@@ -14,6 +14,7 @@
 #define MLPACK_METHODS_ANN_LAYER_LINEAR_HPP
 
 #include <mlpack/prereqs.hpp>
+#include <mlpack/methods/ann/regularizer/lregularizer.hpp>
 
 #include "layer_types.hpp"
 
@@ -31,7 +32,8 @@ namespace ann /** Artificial Neural Network. */ {
  */
 template <
     typename InputDataType = arma::mat,
-    typename OutputDataType = arma::mat
+    typename OutputDataType = arma::mat,
+    typename RegularizerType = L2Regularizer
 >
 class Linear
 {
@@ -45,7 +47,8 @@ class Linear
    * @param inSize The number of input units.
    * @param outSize The number of output units.
    */
-  Linear(const size_t inSize, const size_t outSize);
+  Linear(const size_t inSize, const size_t outSize,
+         RegularizerType regularizer = RegularizerType(0));
 
   /*
    * Reset the layer parameter.
@@ -146,6 +149,9 @@ class Linear
 
   //! Locally-stored output parameter object.
   OutputDataType outputParameter;
+
+  //! Locally-stored regularizer object.
+  RegularizerType regularizer;
 }; // class Linear
 
 } // namespace ann
diff --git a/src/mlpack/methods/ann/layer/linear_impl.hpp b/src/mlpack/methods/ann/layer/linear_impl.hpp
index 92f58351df..d50885ebad 100644
--- a/src/mlpack/methods/ann/layer/linear_impl.hpp
+++ b/src/mlpack/methods/ann/layer/linear_impl.hpp
@@ -19,50 +19,60 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template<typename InputDataType, typename OutputDataType>
-Linear<InputDataType, OutputDataType>::Linear()
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
+Linear<InputDataType, OutputDataType, RegularizerType>::Linear() :
+    inSize(0),
+    outSize(0)
 {
   // Nothing to do here.
 }
 
-template <typename InputDataType, typename OutputDataType>
-Linear<InputDataType, OutputDataType>::Linear(
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
+Linear<InputDataType, OutputDataType, RegularizerType>::Linear(
     const size_t inSize,
-    const size_t outSize) :
+    const size_t outSize,
+    RegularizerType regularizer) :
     inSize(inSize),
-    outSize(outSize)
+    outSize(outSize),
+    regularizer(regularizer)
 {
   weights.set_size(outSize * inSize + outSize, 1);
 }
 
-template<typename InputDataType, typename OutputDataType>
-void Linear<InputDataType, OutputDataType>::Reset()
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
+void Linear<InputDataType, OutputDataType, RegularizerType>::Reset()
 {
   weight = arma::mat(weights.memptr(), outSize, inSize, false, false);
   bias = arma::mat(weights.memptr() + weight.n_elem,
       outSize, 1, false, false);
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 template<typename eT>
-void Linear<InputDataType, OutputDataType>::Forward(
+void Linear<InputDataType, OutputDataType, RegularizerType>::Forward(
     const arma::Mat<eT>&& input, arma::Mat<eT>&& output)
 {
   output = weight * input;
   output.each_col() += bias;
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 template<typename eT>
-void Linear<InputDataType, OutputDataType>::Backward(
+void Linear<InputDataType, OutputDataType, RegularizerType>::Backward(
     const arma::Mat<eT>&& /* input */, arma::Mat<eT>&& gy, arma::Mat<eT>&& g)
 {
   g = weight.t() * gy;
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 template<typename eT>
-void Linear<InputDataType, OutputDataType>::Gradient(
+void Linear<InputDataType, OutputDataType, RegularizerType>::Gradient(
     const arma::Mat<eT>&& input,
     arma::Mat<eT>&& error,
     arma::Mat<eT>&& gradient)
@@ -71,11 +81,13 @@ void Linear<InputDataType, OutputDataType>::Gradient(
       error * input.t());
   gradient.submat(weight.n_elem, 0, gradient.n_elem - 1, 0) =
       arma::sum(error, 1);
+  gradient += regularizer.Evaluate(weights);
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 template<typename Archive>
-void Linear<InputDataType, OutputDataType>::serialize(
+void Linear<InputDataType, OutputDataType, RegularizerType>::serialize(
     Archive& ar, const unsigned int /* version */)
 {
   ar & BOOST_SERIALIZATION_NVP(inSize);
diff --git a/src/mlpack/methods/ann/regularizer/CMakeLists.txt b/src/mlpack/methods/ann/regularizer/CMakeLists.txt
new file mode 100644
index 0000000000..4a897e5267
--- /dev/null
+++ b/src/mlpack/methods/ann/regularizer/CMakeLists.txt
@@ -0,0 +1,15 @@
+# Define the files we need to compile
+# Anything not in this list will not be compiled into mlpack.
+set(SOURCES
+  lregularizer.hpp
+  lregularizer_impl.hpp
+)
+
+# Add directory name to sources.
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+    set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+# Append sources (with directory name) to list of all mlpack sources (used at
+# the parent scope).
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)
diff --git a/src/mlpack/methods/ann/regularizer/lregularizer.hpp b/src/mlpack/methods/ann/regularizer/lregularizer.hpp
new file mode 100644
index 0000000000..5511101308
--- /dev/null
+++ b/src/mlpack/methods/ann/regularizer/lregularizer.hpp
@@ -0,0 +1,78 @@
+/**
+ * @file lregularizer.hpp
+ * @author Saksham Bansal
+ *
+ * Generalized L-regularizer, allowing both l1 and l2 regularization methods.
+ *
+ * This also gives several convenience typedefs for commonly used L-regularizers.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LREGULARIZER_HPP
+#define MLPACK_METHODS_ANN_LREGULARIZER_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann {
+
+/**
+ * The L_p regularizer for arbitrary integer p.
+ *
+ * @tparam Power Power of regularizer; i.e. Power = 1 gives the L1-regularization.
+ */
+template<int TPower>
+class LRegularizer
+{
+ public:
+  /**
+   * Create the regularizer object.
+   *
+   * @param factor The factor for regularization.
+   */
+  LRegularizer(double factor = 1.0);
+
+  /**
+   * Returns the gradient for regularization.
+   *
+   * @tparam VecTypeA Type of weight matrix.
+   * @param a The weight matrix to be regularized.
+   * @return The gradient for the regularization.
+   */
+  template<typename MatType>
+  MatType Evaluate(const MatType& a);
+
+  //! Serialize the regularizer (nothing to do).
+  template<typename Archive>
+  void serialize(Archive& ar, const unsigned int /* version */);
+
+  //! The power of the regularizer.
+  static const int Power = TPower;
+
+  //! The constant for the regularization
+  const int factor;
+};
+
+// Convenience typedefs.
+
+/**
+ * The L1 Regularizer.
+ */
+typedef LRegularizer<1> L1Regularizer;
+
+/**
+ * The L2 Regularizer.
+ */
+typedef LRegularizer<2> L2Regularizer;
+
+
+} // namespace ann
+} // namespace mlpack
+
+// Include implementation.
+#include "lregularizer_impl.hpp"
+
+#endif
diff --git a/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp b/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp
new file mode 100644
index 0000000000..5c3c3ae0c3
--- /dev/null
+++ b/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp
@@ -0,0 +1,64 @@
+/**
+ * @file lregularization_impl.hpp
+ * @author Saksham Bansal
+ *
+ * Implementation of template specializations of LRegularizer class.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LREGULARIZER_IMPL_HPP
+#define MLPACK_METHODS_ANN_LREGULARIZER_IMPL_HPP
+
+// In case it hasn't been included.
+#include "lregularizer.hpp"
+
+namespace mlpack {
+namespace ann {
+
+template<int Power>
+LRegularizer<Power>::LRegularizer(double factor) :
+    factor(factor)
+{
+  // Nothing to do here
+}
+
+
+// Unspecialized implementation. This should almost never be used...
+template<int Power>
+template<typename MatType>
+MatType LRegularizer<Power>::Evaluate(const MatType& a)
+{
+  return arma::pow(a, Power - 1) * Power;
+}
+
+// L1-Regularizer specializations.
+template<>
+template<typename MatType>
+MatType LRegularizer<1>::Evaluate(const MatType& a)
+{
+  return  factor * a/arma::abs(a);
+}
+
+// L2-Regularizer specializations.
+template<>
+template<typename MatType>
+MatType LRegularizer<2>::Evaluate(const MatType& a)
+{
+  return  2 * factor * a;
+}
+
+template<int Power>
+template<typename Archive>
+void LRegularizer<Power>::serialize(
+    Archive& ar, const unsigned int /* version */)
+{
+  ar & BOOST_SERIALIZATION_NVP(factor);
+}
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

From c1bd565e019a6f7fef5c6e8abd2962feeb00556f Mon Sep 17 00:00:00 2001
From: Saksham Bansal <7020962+saksham189@users.noreply.github.com>
Date: Thu, 4 Jul 2019 20:19:33 +0700
Subject: [PATCH 02/11] Add OrthogonalRegularizer, NoRegularizer class Minor
 style fixes

---
 src/mlpack/methods/ann/layer/layer_types.hpp  |  4 +-
 src/mlpack/methods/ann/layer/linear.hpp       | 10 ++-
 src/mlpack/methods/ann/layer/linear_impl.hpp  |  2 +-
 .../methods/ann/regularizer/CMakeLists.txt    |  3 +
 .../methods/ann/regularizer/lregularizer.hpp  | 10 +--
 .../ann/regularizer/lregularizer_impl.hpp     | 12 +--
 .../ann/regularizer/no_regularizer.hpp        | 62 ++++++++++++++++
 .../regularizer/orthogonal_regularizer.hpp    | 74 +++++++++++++++++++
 .../orthogonal_regularizer_impl.hpp           | 65 ++++++++++++++++
 .../methods/ann/regularizer/regularizer.hpp   | 19 +++++
 10 files changed, 243 insertions(+), 18 deletions(-)
 create mode 100644 src/mlpack/methods/ann/regularizer/no_regularizer.hpp
 create mode 100644 src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp
 create mode 100644 src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp
 create mode 100644 src/mlpack/methods/ann/regularizer/regularizer.hpp

diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp
index 8bc2fa5aa3..715529056a 100644
--- a/src/mlpack/methods/ann/layer/layer_types.hpp
+++ b/src/mlpack/methods/ann/layer/layer_types.hpp
@@ -47,7 +47,7 @@
 #include <mlpack/methods/ann/convolution_rules/fft_convolution.hpp>
 
 // Regularizers.
-#include <mlpack/methods/ann/regularizer/lregularizer.hpp>
+#include <mlpack/methods/ann/regularizer/no_regularizer.hpp>
 
 // Loss function modules.
 #include <mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp>
@@ -184,7 +184,7 @@ using LayerTypes = boost::variant<
     LayerNorm<arma::mat, arma::mat>*,
     LeakyReLU<arma::mat, arma::mat>*,
     CReLU<arma::mat, arma::mat>*,
-    Linear<arma::mat, arma::mat, L2Regularizer>*,
+    Linear<arma::mat, arma::mat, NoRegularizer>*,
     LinearNoBias<arma::mat, arma::mat>*,
     LogSoftMax<arma::mat, arma::mat>*,
     Lookup<arma::mat, arma::mat>*,
diff --git a/src/mlpack/methods/ann/layer/linear.hpp b/src/mlpack/methods/ann/layer/linear.hpp
index b73f242208..b5b9267cc8 100644
--- a/src/mlpack/methods/ann/layer/linear.hpp
+++ b/src/mlpack/methods/ann/layer/linear.hpp
@@ -14,7 +14,8 @@
 #define MLPACK_METHODS_ANN_LAYER_LINEAR_HPP
 
 #include <mlpack/prereqs.hpp>
-#include <mlpack/methods/ann/regularizer/lregularizer.hpp>
+#include <mlpack/methods/ann/regularizer/no_regularizer.hpp>
+#include <mlpack/methods/ann/regularizer/regularizer.hpp>
 
 #include "layer_types.hpp"
 
@@ -33,7 +34,7 @@ namespace ann /** Artificial Neural Network. */ {
 template <
     typename InputDataType = arma::mat,
     typename OutputDataType = arma::mat,
-    typename RegularizerType = L2Regularizer
+    typename RegularizerType = NoRegularizer
 >
 class Linear
 {
@@ -47,8 +48,9 @@ class Linear
    * @param inSize The number of input units.
    * @param outSize The number of output units.
    */
-  Linear(const size_t inSize, const size_t outSize,
-         RegularizerType regularizer = RegularizerType(0));
+  Linear(const size_t inSize,
+         const size_t outSize,
+         RegularizerType regularizer = RegularizerType());
 
   /*
    * Reset the layer parameter.
diff --git a/src/mlpack/methods/ann/layer/linear_impl.hpp b/src/mlpack/methods/ann/layer/linear_impl.hpp
index d50885ebad..127d789716 100644
--- a/src/mlpack/methods/ann/layer/linear_impl.hpp
+++ b/src/mlpack/methods/ann/layer/linear_impl.hpp
@@ -81,7 +81,7 @@ void Linear<InputDataType, OutputDataType, RegularizerType>::Gradient(
       error * input.t());
   gradient.submat(weight.n_elem, 0, gradient.n_elem - 1, 0) =
       arma::sum(error, 1);
-  gradient += regularizer.Evaluate(weights);
+  regularizer.Evaluate(weights, gradient);
 }
 
 template<typename InputDataType, typename OutputDataType,
diff --git a/src/mlpack/methods/ann/regularizer/CMakeLists.txt b/src/mlpack/methods/ann/regularizer/CMakeLists.txt
index 4a897e5267..94d1a253cb 100644
--- a/src/mlpack/methods/ann/regularizer/CMakeLists.txt
+++ b/src/mlpack/methods/ann/regularizer/CMakeLists.txt
@@ -3,6 +3,9 @@
 set(SOURCES
   lregularizer.hpp
   lregularizer_impl.hpp
+  orthogonal_regularizer.hpp
+  orthogonal_regularizer_impl.hpp
+  no_regularizer.hpp
 )
 
 # Add directory name to sources.
diff --git a/src/mlpack/methods/ann/regularizer/lregularizer.hpp b/src/mlpack/methods/ann/regularizer/lregularizer.hpp
index 5511101308..6227e02376 100644
--- a/src/mlpack/methods/ann/regularizer/lregularizer.hpp
+++ b/src/mlpack/methods/ann/regularizer/lregularizer.hpp
@@ -36,14 +36,14 @@ class LRegularizer
   LRegularizer(double factor = 1.0);
 
   /**
-   * Returns the gradient for regularization.
+   * Calculate the gradient for regularization.
    *
-   * @tparam VecTypeA Type of weight matrix.
-   * @param a The weight matrix to be regularized.
-   * @return The gradient for the regularization.
+   * @tparam MatType Type of weight matrix.
+   * @param weight The weight matrix to be regularized.
+   * @param gradient The calculated gradient.
    */
   template<typename MatType>
-  MatType Evaluate(const MatType& a);
+  void Evaluate(const MatType& weight, MatType& gradient);
 
   //! Serialize the regularizer (nothing to do).
   template<typename Archive>
diff --git a/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp b/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp
index 5c3c3ae0c3..0a391e2de1 100644
--- a/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp
+++ b/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp
@@ -29,25 +29,25 @@ LRegularizer<Power>::LRegularizer(double factor) :
 // Unspecialized implementation. This should almost never be used...
 template<int Power>
 template<typename MatType>
-MatType LRegularizer<Power>::Evaluate(const MatType& a)
+void LRegularizer<Power>::Evaluate(const MatType& weight, MatType& gradient)
 {
-  return arma::pow(a, Power - 1) * Power;
+  gradient = arma::vectorise(arma::pow(weight, Power - 1) * Power);
 }
 
 // L1-Regularizer specializations.
 template<>
 template<typename MatType>
-MatType LRegularizer<1>::Evaluate(const MatType& a)
+void LRegularizer<1>::Evaluate(const MatType& weight, MatType& gradient)
 {
-  return  factor * a/arma::abs(a);
+  gradient += arma::vectorise(factor * weight / arma::abs(weight));
 }
 
 // L2-Regularizer specializations.
 template<>
 template<typename MatType>
-MatType LRegularizer<2>::Evaluate(const MatType& a)
+void LRegularizer<2>::Evaluate(const MatType& weight, MatType& gradient)
 {
-  return  2 * factor * a;
+  gradient = arma::vectorise(2 * factor * weight);
 }
 
 template<int Power>
diff --git a/src/mlpack/methods/ann/regularizer/no_regularizer.hpp b/src/mlpack/methods/ann/regularizer/no_regularizer.hpp
new file mode 100644
index 0000000000..591e8534fc
--- /dev/null
+++ b/src/mlpack/methods/ann/regularizer/no_regularizer.hpp
@@ -0,0 +1,62 @@
+/**
+ * @file no_regularizer.hpp
+ * @author Saksham Bansal
+ *
+ * Definition of the NoRegularizer class.
+ *
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_NOREGULARIZER_HPP
+#define MLPACK_METHODS_ANN_NOREGULARIZER_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann {
+
+/**
+ * Implementation of the NoRegularizer. This does not add any
+ * regularization to the weights.
+ */
+class NoRegularizer
+{
+ public:
+  /**
+   * Create the regularizer object.
+   *
+   * @param factor The factor for regularization.
+   */
+  NoRegularizer()
+  {
+    // Nothing to do here.
+  };
+
+  /**
+   * Calculate the gradient for regularization.
+   *
+   * @tparam MatType Type of weight matrix.
+   * @param weight The weight matrix to be regularized.
+   * @param gradient The calculated gradient.
+   */
+  template<typename MatType>
+  void Evaluate(const MatType& weight, MatType& gradient)
+  {
+    // Nothing to do here.
+  }
+
+  //! Serialize the regularizer (nothing to do).
+  template<typename Archive>
+  void serialize(Archive& ar, const unsigned int /* version */)
+  {
+    // Nothing to do here.
+  }
+};
+
+} // namespace ann
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp
new file mode 100644
index 0000000000..443fa2010a
--- /dev/null
+++ b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp
@@ -0,0 +1,74 @@
+/**
+ * @file orthogonal_regularizer.hpp
+ * @author Saksham Bansal
+ *
+ * Definition of the OrthogonalRegularizer class.
+ *
+ * This also gives several convenience typedefs for commonly used L-regularizers.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LREGULARIZER_HPP
+#define MLPACK_METHODS_ANN_LREGULARIZER_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann {
+
+/**
+ * Implementation of the OrthogonalRegularizer. Orthogonality of weights is a
+ * desirable property because multiplication by an orthogonal matrix leaves
+ * the norm of the matrix unchanged. The orthogonal regularization technique
+ * encourages weights to be orthogonal.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @inproceedings{WanICML2013,
+ *   title={Neural Photo Editing with Introspective Adversarial Networks},
+ *   booktitle = {5th International Conference on Learning Representations
+ *                (ICLR - 17)},
+ *   author = {Andrew Brock and Theodore Lim and J.M. Ritchie and Nick Weston},
+ *   year = {2017}
+ * }
+ * @endcode
+ */
+class OrthogonalRegularizer
+{
+ public:
+  /**
+   * Create the regularizer object.
+   *
+   * @param factor The factor for regularization.
+   */
+  OrthogonalRegularizer(double factor = 1.0);
+
+  /**
+   * Calculate the gradient for regularization.
+   *
+   * @tparam MatType Type of weight matrix.
+   * @param weight The weight matrix to be regularized.
+   * @param gradient The calculated gradient.
+   */
+  template<typename MatType>
+  void Evaluate(const MatType& weight, MatType& gradient);
+
+  //! Serialize the regularizer (nothing to do).
+  template<typename Archive>
+  void serialize(Archive& ar, const unsigned int /* version */);
+
+  //! The constant for the regularization
+  const int factor;
+};
+
+} // namespace ann
+} // namespace mlpack
+
+// Include implementation.
+#include "orthogonal_regularizer_impl.hpp"
+
+#endif
diff --git a/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp
new file mode 100644
index 0000000000..3b68ce2225
--- /dev/null
+++ b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp
@@ -0,0 +1,65 @@
+/**
+ * @file orthogonal_regularizer_impl.hpp
+ * @author Saksham Bansal
+ *
+ * Implementation of OrthogonalRegularizer class.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LREGULARIZER_IMPL_HPP
+#define MLPACK_METHODS_ANN_LREGULARIZER_IMPL_HPP
+
+// In case it hasn't been included.
+#include "orthogonal_regularizer.hpp"
+
+namespace mlpack {
+namespace ann {
+
+OrthogonalRegularizer::OrthogonalRegularizer(double factor) :
+    factor(factor)
+{
+  // Nothing to do here.
+}
+
+template<typename MatType>
+void OrthogonalRegularizer::Evaluate(const MatType& weight, MatType& gradient)
+{
+  arma::mat grad = arma::mat(arma::size(weight), arma::fill::zeros);
+
+  for (size_t i = 0; i < weight.n_rows; i++)
+  {
+    for (size_t j = 0; j < weight.n_rows; j++)
+    {
+      if (i == j)
+      {
+        int s =
+            arma::as_scalar(
+            arma::sign((weight.row(i) * weight.row(i).t()) - 1));
+        grad.row(i) += 2 * s * weight.row(i);
+      }
+      else
+      {
+        int s = arma::as_scalar(arma::sign(weight.row(i) * weight.row(j).t()));
+        grad.row(i) += s * weight.row(j);
+        grad.row(j) += s * weight.row(i);
+      }
+    }
+  }
+  gradient += arma::vectorise(grad) * factor;
+}
+
+template<int Power>
+template<typename Archive>
+void OrthogonalRegularizer<Power>::serialize(
+    Archive& ar, const unsigned int /* version */)
+{
+  ar & BOOST_SERIALIZATION_NVP(factor);
+}
+
+} // namespace ann
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/methods/ann/regularizer/regularizer.hpp b/src/mlpack/methods/ann/regularizer/regularizer.hpp
new file mode 100644
index 0000000000..035350563b
--- /dev/null
+++ b/src/mlpack/methods/ann/regularizer/regularizer.hpp
@@ -0,0 +1,19 @@
+/**
+ * @file regularizer.hpp
+ * @author Saksham Bansal
+ *
+ * This includes various regularizers to construct a model.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_REGULARIZER_LAYER_HPP
+#define MLPACK_METHODS_ANN_REGULARIZER_LAYER_HPP
+
+#include "lregularizer.hpp"
+#include "orthogonal_regularizer.hpp"
+#include "no_regularizer.hpp"
+
+#endif

From 9a0e3e0f2989e43da40d43cd1512266b2ca79e09 Mon Sep 17 00:00:00 2001
From: Saksham Bansal <7020962+saksham189@users.noreply.github.com>
Date: Tue, 30 Jul 2019 17:13:14 +0530
Subject: [PATCH 03/11] Minor Style fix

---
 src/mlpack/methods/ann/layer/linear.hpp                         | 1 -
 src/mlpack/methods/ann/regularizer/lregularizer.hpp             | 2 --
 src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp        | 1 -
 src/mlpack/methods/ann/regularizer/no_regularizer.hpp           | 1 -
 src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp   | 2 --
 .../methods/ann/regularizer/orthogonal_regularizer_impl.hpp     | 1 +
 6 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/linear.hpp b/src/mlpack/methods/ann/layer/linear.hpp
index b5b9267cc8..02e3c16acf 100644
--- a/src/mlpack/methods/ann/layer/linear.hpp
+++ b/src/mlpack/methods/ann/layer/linear.hpp
@@ -14,7 +14,6 @@
 #define MLPACK_METHODS_ANN_LAYER_LINEAR_HPP
 
 #include <mlpack/prereqs.hpp>
-#include <mlpack/methods/ann/regularizer/no_regularizer.hpp>
 #include <mlpack/methods/ann/regularizer/regularizer.hpp>
 
 #include "layer_types.hpp"
diff --git a/src/mlpack/methods/ann/regularizer/lregularizer.hpp b/src/mlpack/methods/ann/regularizer/lregularizer.hpp
index 6227e02376..edd3e78d22 100644
--- a/src/mlpack/methods/ann/regularizer/lregularizer.hpp
+++ b/src/mlpack/methods/ann/regularizer/lregularizer.hpp
@@ -57,7 +57,6 @@ class LRegularizer
 };
 
 // Convenience typedefs.
-
 /**
  * The L1 Regularizer.
  */
@@ -68,7 +67,6 @@ typedef LRegularizer<1> L1Regularizer;
  */
 typedef LRegularizer<2> L2Regularizer;
 
-
 } // namespace ann
 } // namespace mlpack
 
diff --git a/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp b/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp
index 0a391e2de1..38bed706ac 100644
--- a/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp
+++ b/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp
@@ -25,7 +25,6 @@ LRegularizer<Power>::LRegularizer(double factor) :
   // Nothing to do here
 }
 
-
 // Unspecialized implementation. This should almost never be used...
 template<int Power>
 template<typename MatType>
diff --git a/src/mlpack/methods/ann/regularizer/no_regularizer.hpp b/src/mlpack/methods/ann/regularizer/no_regularizer.hpp
index 591e8534fc..e813d8381d 100644
--- a/src/mlpack/methods/ann/regularizer/no_regularizer.hpp
+++ b/src/mlpack/methods/ann/regularizer/no_regularizer.hpp
@@ -4,7 +4,6 @@
  *
  * Definition of the NoRegularizer class.
  *
- *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
  * 3-clause BSD license along with mlpack.  If not, see
diff --git a/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp
index 443fa2010a..d99940de48 100644
--- a/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp
+++ b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp
@@ -4,8 +4,6 @@
  *
  * Definition of the OrthogonalRegularizer class.
  *
- * This also gives several convenience typedefs for commonly used L-regularizers.
- *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
  * 3-clause BSD license along with mlpack.  If not, see
diff --git a/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp
index 3b68ce2225..2e41aa9c5d 100644
--- a/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp
+++ b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp
@@ -48,6 +48,7 @@ void OrthogonalRegularizer::Evaluate(const MatType& weight, MatType& gradient)
       }
     }
   }
+
   gradient += arma::vectorise(grad) * factor;
 }
 

From c3880c3109a55b18e92f4c32668adeb334fdbc27 Mon Sep 17 00:00:00 2001
From: Saksham Bansal <7020962+saksham189@users.noreply.github.com>
Date: Thu, 1 Aug 2019 17:51:52 +0530
Subject: [PATCH 04/11] Linear No Bias add regularizers

---
 src/mlpack/methods/ann/layer/layer_types.hpp  |  8 +++-
 .../methods/ann/layer/linear_no_bias.hpp      | 11 ++++-
 .../methods/ann/layer/linear_no_bias_impl.hpp | 45 ++++++++++++-------
 3 files changed, 44 insertions(+), 20 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp
index 715529056a..4ad46ddd76 100644
--- a/src/mlpack/methods/ann/layer/layer_types.hpp
+++ b/src/mlpack/methods/ann/layer/layer_types.hpp
@@ -59,7 +59,6 @@ template<typename InputDataType, typename OutputDataType> class BatchNorm;
 template<typename InputDataType, typename OutputDataType> class DropConnect;
 template<typename InputDataType, typename OutputDataType> class Glimpse;
 template<typename InputDataType, typename OutputDataType> class LayerNorm;
-template<typename InputDataType, typename OutputDataType> class LinearNoBias;
 template<typename InputDataType, typename OutputDataType> class LSTM;
 template<typename InputDataType, typename OutputDataType> class GRU;
 template<typename InputDataType, typename OutputDataType> class FastLSTM;
@@ -71,6 +70,11 @@ template<typename InputDataType,
          typename RegularizerType>
 class Linear;
 
+template<typename InputDataType,
+         typename OutputDataType,
+         typename RegularizerType>
+class LinearNoBias;
+
 template<typename InputDataType,
          typename OutputDataType
 >
@@ -185,7 +189,7 @@ using LayerTypes = boost::variant<
     LeakyReLU<arma::mat, arma::mat>*,
     CReLU<arma::mat, arma::mat>*,
     Linear<arma::mat, arma::mat, NoRegularizer>*,
-    LinearNoBias<arma::mat, arma::mat>*,
+    LinearNoBias<arma::mat, arma::mat, NoRegularizer>*,
     LogSoftMax<arma::mat, arma::mat>*,
     Lookup<arma::mat, arma::mat>*,
     LSTM<arma::mat, arma::mat>*,
diff --git a/src/mlpack/methods/ann/layer/linear_no_bias.hpp b/src/mlpack/methods/ann/layer/linear_no_bias.hpp
index d171e2c2de..33e89830c2 100644
--- a/src/mlpack/methods/ann/layer/linear_no_bias.hpp
+++ b/src/mlpack/methods/ann/layer/linear_no_bias.hpp
@@ -14,6 +14,7 @@
 #define MLPACK_METHODS_ANN_LAYER_LINEAR_NO_BIAS_HPP
 
 #include <mlpack/prereqs.hpp>
+#include <mlpack/methods/ann/regularizer/regularizer.hpp>
 
 #include "layer_types.hpp"
 
@@ -31,7 +32,8 @@ namespace ann /** Artificial Neural Network. */ {
  */
 template <
     typename InputDataType = arma::mat,
-    typename OutputDataType = arma::mat
+    typename OutputDataType = arma::mat,
+    typename RegularizerType = NoRegularizer
 >
 class LinearNoBias
 {
@@ -44,7 +46,9 @@ class LinearNoBias
    * @param inSize The number of input units.
    * @param outSize The number of output units.
    */
-  LinearNoBias(const size_t inSize, const size_t outSize);
+  LinearNoBias(const size_t inSize,
+               const size_t outSize,
+               RegularizerType regularizer = RegularizerType());
 
   /*
    * Reset the layer parameter.
@@ -142,6 +146,9 @@ class LinearNoBias
 
   //! Locally-stored output parameter object.
   OutputDataType outputParameter;
+
+  //! Locally-stored regularizer object.
+  RegularizerType regularizer;
 }; // class LinearNoBias
 
 } // namespace ann
diff --git a/src/mlpack/methods/ann/layer/linear_no_bias_impl.hpp b/src/mlpack/methods/ann/layer/linear_no_bias_impl.hpp
index 42070c2d30..41e432da17 100644
--- a/src/mlpack/methods/ann/layer/linear_no_bias_impl.hpp
+++ b/src/mlpack/methods/ann/layer/linear_no_bias_impl.hpp
@@ -19,57 +19,70 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template<typename InputDataType, typename OutputDataType>
-LinearNoBias<InputDataType, OutputDataType>::LinearNoBias()
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
+LinearNoBias<InputDataType, OutputDataType, RegularizerType>::LinearNoBias() :
+    inSize(0),
+    outSize(0)
 {
   // Nothing to do here.
 }
 
-template<typename InputDataType, typename OutputDataType>
-LinearNoBias<InputDataType, OutputDataType>::LinearNoBias(
-    const size_t inSize, const size_t outSize) :
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
+LinearNoBias<InputDataType, OutputDataType, RegularizerType>::LinearNoBias(
+    const size_t inSize,
+    const size_t outSize,
+    RegularizerType regularizer) :
     inSize(inSize),
-    outSize(outSize)
+    outSize(outSize),
+    regularizer(regularizer)
 {
   weights.set_size(outSize * inSize, 1);
 }
 
-template <typename InputDataType, typename OutputDataType>
-void LinearNoBias<InputDataType, OutputDataType>::Reset()
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
+void LinearNoBias<InputDataType, OutputDataType, RegularizerType>::Reset()
 {
   weight = arma::mat(weights.memptr(), outSize, inSize, false, false);
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 template<typename eT>
-void LinearNoBias<InputDataType, OutputDataType>::Forward(
+void LinearNoBias<InputDataType, OutputDataType, RegularizerType>::Forward(
     const arma::Mat<eT>&& input, arma::Mat<eT>&& output)
 {
   output = weight * input;
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 template<typename eT>
-void LinearNoBias<InputDataType, OutputDataType>::Backward(
+void LinearNoBias<InputDataType, OutputDataType, RegularizerType>::Backward(
     const arma::Mat<eT>&& /* input */, arma::Mat<eT>&& gy, arma::Mat<eT>&& g)
 {
   g = weight.t() * gy;
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 template<typename eT>
-void LinearNoBias<InputDataType, OutputDataType>::Gradient(
+void LinearNoBias<InputDataType, OutputDataType, RegularizerType>::Gradient(
     const arma::Mat<eT>&& input,
     arma::Mat<eT>&& error,
     arma::Mat<eT>&& gradient)
 {
   gradient.submat(0, 0, weight.n_elem - 1, 0) = arma::vectorise(
       error * input.t());
+  regularizer.Evaluate(weights, gradient);
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 template<typename Archive>
-void LinearNoBias<InputDataType, OutputDataType>::serialize(
+void LinearNoBias<InputDataType, OutputDataType, RegularizerType>::serialize(
     Archive& ar, const unsigned int /* version */)
 {
   ar & BOOST_SERIALIZATION_NVP(inSize);

From bbe88e7c9bc1468daf9ce708adf51994fcf1a0dc Mon Sep 17 00:00:00 2001
From: Saksham Bansal <7020962+saksham189@users.noreply.github.com>
Date: Mon, 5 Aug 2019 14:40:00 +0530
Subject: [PATCH 05/11] Style fix

---
 src/mlpack/methods/ann/regularizer/CMakeLists.txt        | 2 +-
 src/mlpack/methods/ann/regularizer/lregularizer.hpp      | 1 -
 src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp | 2 +-
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/mlpack/methods/ann/regularizer/CMakeLists.txt b/src/mlpack/methods/ann/regularizer/CMakeLists.txt
index 94d1a253cb..1a608b9b40 100644
--- a/src/mlpack/methods/ann/regularizer/CMakeLists.txt
+++ b/src/mlpack/methods/ann/regularizer/CMakeLists.txt
@@ -1,4 +1,4 @@
-# Define the files we need to compile
+# Define the files we need to compile.
 # Anything not in this list will not be compiled into mlpack.
 set(SOURCES
   lregularizer.hpp
diff --git a/src/mlpack/methods/ann/regularizer/lregularizer.hpp b/src/mlpack/methods/ann/regularizer/lregularizer.hpp
index edd3e78d22..6749481b77 100644
--- a/src/mlpack/methods/ann/regularizer/lregularizer.hpp
+++ b/src/mlpack/methods/ann/regularizer/lregularizer.hpp
@@ -3,7 +3,6 @@
  * @author Saksham Bansal
  *
  * Generalized L-regularizer, allowing both l1 and l2 regularization methods.
- *
  * This also gives several convenience typedefs for commonly used L-regularizers.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
diff --git a/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp b/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp
index 38bed706ac..b9bbce13d4 100644
--- a/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp
+++ b/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp
@@ -30,7 +30,7 @@ template<int Power>
 template<typename MatType>
 void LRegularizer<Power>::Evaluate(const MatType& weight, MatType& gradient)
 {
-  gradient = arma::vectorise(arma::pow(weight, Power - 1) * Power);
+  gradient = arma::vectorise(arma::pow(weight, Power - 1) * Power * factor);
 }
 
 // L1-Regularizer specializations.

From 470f4253896e32041314516fa50bfac747cc6980 Mon Sep 17 00:00:00 2001
From: Saksham Bansal <7020962+saksham189@users.noreply.github.com>
Date: Tue, 6 Aug 2019 01:27:12 +0530
Subject: [PATCH 06/11] Add tests

---
 .../methods/ann/regularizer/lregularizer.hpp  |  2 +-
 .../ann/regularizer/lregularizer_impl.hpp     |  2 +-
 src/mlpack/tests/CMakeLists.txt               |  1 +
 src/mlpack/tests/ann_regularizer_test.cpp     | 87 +++++++++++++++++++
 src/mlpack/tests/ann_test_tools.hpp           | 33 +++++++
 5 files changed, 123 insertions(+), 2 deletions(-)
 create mode 100644 src/mlpack/tests/ann_regularizer_test.cpp

diff --git a/src/mlpack/methods/ann/regularizer/lregularizer.hpp b/src/mlpack/methods/ann/regularizer/lregularizer.hpp
index 6749481b77..3b7587e70b 100644
--- a/src/mlpack/methods/ann/regularizer/lregularizer.hpp
+++ b/src/mlpack/methods/ann/regularizer/lregularizer.hpp
@@ -52,7 +52,7 @@ class LRegularizer
   static const int Power = TPower;
 
   //! The constant for the regularization
-  const int factor;
+  const double factor;
 };
 
 // Convenience typedefs.
diff --git a/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp b/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp
index b9bbce13d4..6c07484ace 100644
--- a/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp
+++ b/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp
@@ -38,7 +38,7 @@ template<>
 template<typename MatType>
 void LRegularizer<1>::Evaluate(const MatType& weight, MatType& gradient)
 {
-  gradient += arma::vectorise(factor * weight / arma::abs(weight));
+  gradient = arma::vectorise(factor * weight / arma::abs(weight));
 }
 
 // L2-Regularizer specializations.
diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt
index 15056a6878..bd2db1c078 100644
--- a/src/mlpack/tests/CMakeLists.txt
+++ b/src/mlpack/tests/CMakeLists.txt
@@ -6,6 +6,7 @@ add_executable(mlpack_test
   aknn_test.cpp
   ann_dist_test.cpp
   ann_layer_test.cpp
+  ann_regularizer_test.cpp
   ann_test_tools.hpp
   arma_extend_test.cpp
   armadillo_svd_test.cpp
diff --git a/src/mlpack/tests/ann_regularizer_test.cpp b/src/mlpack/tests/ann_regularizer_test.cpp
new file mode 100644
index 0000000000..10efdf05cd
--- /dev/null
+++ b/src/mlpack/tests/ann_regularizer_test.cpp
@@ -0,0 +1,87 @@
+/**
+ * @file ann_regularizer_test.cpp
+ * @author Saksham Bansal
+ *
+ * Tests the ann regularizer modules.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include <mlpack/core.hpp>
+
+#include <mlpack/methods/ann/layer/layer.hpp>
+#include <mlpack/methods/ann/layer/layer_types.hpp>
+#include <mlpack/methods/ann/init_rules/random_init.hpp>
+#include <mlpack/methods/ann/regularizer/regularizer.hpp>
+
+#include <boost/test/unit_test.hpp>
+#include "test_tools.hpp"
+#include "ann_test_tools.hpp"
+#include "serialization.hpp"
+
+using namespace mlpack;
+using namespace mlpack::ann;
+
+BOOST_AUTO_TEST_SUITE(ANNRegularizerTest);
+
+BOOST_AUTO_TEST_CASE(GradientL1RegularizerTest)
+{
+  // Add function gradient instantiation.
+  struct GradientFunction
+  {
+    GradientFunction() :
+      factor(0.6),
+      reg(factor)
+    {
+      // Nothing to do here.
+    }
+
+    double Output(const arma::mat& weight, size_t i, size_t j)
+    {
+     return abs(weight(i, j)) * factor;
+    }
+
+    void Gradient(arma::mat& weight, arma::mat& gradient)
+    {
+      reg.Evaluate(weight, gradient);
+    }
+    double factor;
+    L1Regularizer reg;
+
+  } function;
+
+  BOOST_REQUIRE_LE(CheckRegularizerGradient(function), 1e-4);
+}
+
+BOOST_AUTO_TEST_CASE(GradientL2RegularizerTest)
+{
+  // Add function gradient instantiation.
+  struct GradientFunction
+  {
+    GradientFunction() :
+      factor(0.6),
+      reg(factor)
+    {
+      // Nothing to do here.
+    }
+
+    double Output(const arma::mat& weight, size_t i, size_t j)
+    {
+     return weight(i, j) * weight(i, j) * factor;
+    }
+
+    void Gradient(arma::mat& weight, arma::mat& gradient)
+    {
+      reg.Evaluate(weight, gradient);
+    }
+    double factor;
+    L2Regularizer reg;
+
+  } function;
+
+  BOOST_REQUIRE_LE(CheckRegularizerGradient(function), 1e-4);
+}
+
+BOOST_AUTO_TEST_SUITE_END();
diff --git a/src/mlpack/tests/ann_test_tools.hpp b/src/mlpack/tests/ann_test_tools.hpp
index 67f7108cd0..f91cd1e433 100644
--- a/src/mlpack/tests/ann_test_tools.hpp
+++ b/src/mlpack/tests/ann_test_tools.hpp
@@ -167,4 +167,37 @@ double CheckGradient(FunctionType& function, const double eps = 1e-7)
       arma::norm(orgGradient + estGradient);
 }
 
+// Simple numerical gradient checker for regularizers.
+template<class FunctionType>
+double CheckRegularizerGradient(FunctionType& function, const double eps = 1e-7)
+{
+  // Get gradients for the current parameters.
+  arma::mat orgGradient, estGradient;
+  arma::mat weight = arma::randu(10, 15);
+  function.Gradient(weight, orgGradient);
+
+  estGradient = arma::zeros(weight.n_rows, weight.n_cols);
+
+  // Compute numeric approximations to gradient.
+  for (size_t i = 0; i < weight.n_rows; ++i)
+  {
+    for (size_t j = 0; j < weight.n_cols; ++j)
+    {
+      weight(i, j) += eps;
+      double costPlus = function.Output(weight, i, j);
+      weight(i, j) -= 2 * eps;
+      double costMinus = function.Output(weight, i, j);
+
+      // Restore the weight value.
+      weight(i, j) += eps;
+      estGradient(i, j) = (costPlus - costMinus) / (2 * eps);
+    }
+  }
+  estGradient = arma::vectorise(estGradient);
+
+  // Estimate error of gradient.
+  return arma::norm(orgGradient - estGradient) /
+      arma::norm(orgGradient + estGradient);
+}
+
 #endif

From ab199df979cdb6c55323bbcde50b01354d576496 Mon Sep 17 00:00:00 2001
From: Saksham Bansal <7020962+saksham189@users.noreply.github.com>
Date: Tue, 6 Aug 2019 01:36:06 +0530
Subject: [PATCH 07/11] Style fix

---
 src/mlpack/methods/ann/regularizer/lregularizer.hpp   |  2 +-
 src/mlpack/methods/ann/regularizer/no_regularizer.hpp | 11 +----------
 .../ann/regularizer/orthogonal_regularizer.hpp        |  2 +-
 src/mlpack/tests/ann_regularizer_test.cpp             |  6 +++---
 src/mlpack/tests/ann_test_tools.hpp                   |  4 ++--
 5 files changed, 8 insertions(+), 17 deletions(-)

diff --git a/src/mlpack/methods/ann/regularizer/lregularizer.hpp b/src/mlpack/methods/ann/regularizer/lregularizer.hpp
index 3b7587e70b..1cca332887 100644
--- a/src/mlpack/methods/ann/regularizer/lregularizer.hpp
+++ b/src/mlpack/methods/ann/regularizer/lregularizer.hpp
@@ -52,7 +52,7 @@ class LRegularizer
   static const int Power = TPower;
 
   //! The constant for the regularization
-  const double factor;
+  double factor;
 };
 
 // Convenience typedefs.
diff --git a/src/mlpack/methods/ann/regularizer/no_regularizer.hpp b/src/mlpack/methods/ann/regularizer/no_regularizer.hpp
index e813d8381d..f411816f18 100644
--- a/src/mlpack/methods/ann/regularizer/no_regularizer.hpp
+++ b/src/mlpack/methods/ann/regularizer/no_regularizer.hpp
@@ -26,8 +26,6 @@ class NoRegularizer
  public:
   /**
    * Create the regularizer object.
-   *
-   * @param factor The factor for regularization.
    */
   NoRegularizer()
   {
@@ -42,14 +40,7 @@ class NoRegularizer
    * @param gradient The calculated gradient.
    */
   template<typename MatType>
-  void Evaluate(const MatType& weight, MatType& gradient)
-  {
-    // Nothing to do here.
-  }
-
-  //! Serialize the regularizer (nothing to do).
-  template<typename Archive>
-  void serialize(Archive& ar, const unsigned int /* version */)
+  void Evaluate(const MatType& /* weight */, MatType& /* gradient */)
   {
     // Nothing to do here.
   }
diff --git a/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp
index d99940de48..83dde5563a 100644
--- a/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp
+++ b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp
@@ -60,7 +60,7 @@ class OrthogonalRegularizer
   void serialize(Archive& ar, const unsigned int /* version */);
 
   //! The constant for the regularization
-  const int factor;
+  double factor;
 };
 
 } // namespace ann
diff --git a/src/mlpack/tests/ann_regularizer_test.cpp b/src/mlpack/tests/ann_regularizer_test.cpp
index 10efdf05cd..9d60f9deea 100644
--- a/src/mlpack/tests/ann_regularizer_test.cpp
+++ b/src/mlpack/tests/ann_regularizer_test.cpp
@@ -40,16 +40,16 @@ BOOST_AUTO_TEST_CASE(GradientL1RegularizerTest)
 
     double Output(const arma::mat& weight, size_t i, size_t j)
     {
-     return abs(weight(i, j)) * factor;
+     return std::abs(weight(i, j)) * factor;
     }
 
     void Gradient(arma::mat& weight, arma::mat& gradient)
     {
       reg.Evaluate(weight, gradient);
     }
+
     double factor;
     L1Regularizer reg;
-
   } function;
 
   BOOST_REQUIRE_LE(CheckRegularizerGradient(function), 1e-4);
@@ -76,9 +76,9 @@ BOOST_AUTO_TEST_CASE(GradientL2RegularizerTest)
     {
       reg.Evaluate(weight, gradient);
     }
+
     double factor;
     L2Regularizer reg;
-
   } function;
 
   BOOST_REQUIRE_LE(CheckRegularizerGradient(function), 1e-4);
diff --git a/src/mlpack/tests/ann_test_tools.hpp b/src/mlpack/tests/ann_test_tools.hpp
index f91cd1e433..2c8a9dbf86 100644
--- a/src/mlpack/tests/ann_test_tools.hpp
+++ b/src/mlpack/tests/ann_test_tools.hpp
@@ -185,7 +185,7 @@ double CheckRegularizerGradient(FunctionType& function, const double eps = 1e-7)
     {
       weight(i, j) += eps;
       double costPlus = function.Output(weight, i, j);
-      weight(i, j) -= 2 * eps;
+      weight(i, j) -= (2 * eps);
       double costMinus = function.Output(weight, i, j);
 
       // Restore the weight value.
@@ -193,8 +193,8 @@ double CheckRegularizerGradient(FunctionType& function, const double eps = 1e-7)
       estGradient(i, j) = (costPlus - costMinus) / (2 * eps);
     }
   }
-  estGradient = arma::vectorise(estGradient);
 
+  estGradient = arma::vectorise(estGradient);
   // Estimate error of gradient.
   return arma::norm(orgGradient - estGradient) /
       arma::norm(orgGradient + estGradient);

From 0f6f1528d6b9bddb1e8b31d92a574ad2cf3c388a Mon Sep 17 00:00:00 2001
From: Saksham Bansal <7020962+saksham189@users.noreply.github.com>
Date: Wed, 7 Aug 2019 23:34:48 +0530
Subject: [PATCH 08/11] Orthogonal Regularization test

---
 src/mlpack/methods/ann/layer/linear.hpp       |  2 +-
 .../methods/ann/layer/linear_no_bias.hpp      |  2 +-
 .../ann/regularizer/no_regularizer.hpp        |  4 +--
 .../regularizer/orthogonal_regularizer.hpp    |  4 +--
 .../orthogonal_regularizer_impl.hpp           |  9 +++--
 .../methods/ann/regularizer/regularizer.hpp   |  4 +--
 src/mlpack/tests/ann_regularizer_test.cpp     | 33 ++++++++++++++++++-
 src/mlpack/tests/ann_test_tools.hpp           |  6 ++--
 8 files changed, 48 insertions(+), 16 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/linear.hpp b/src/mlpack/methods/ann/layer/linear.hpp
index 02e3c16acf..74c709adfb 100644
--- a/src/mlpack/methods/ann/layer/linear.hpp
+++ b/src/mlpack/methods/ann/layer/linear.hpp
@@ -14,7 +14,7 @@
 #define MLPACK_METHODS_ANN_LAYER_LINEAR_HPP
 
 #include <mlpack/prereqs.hpp>
-#include <mlpack/methods/ann/regularizer/regularizer.hpp>
+#include <mlpack/methods/ann/regularizer/no_regularizer.hpp>
 
 #include "layer_types.hpp"
 
diff --git a/src/mlpack/methods/ann/layer/linear_no_bias.hpp b/src/mlpack/methods/ann/layer/linear_no_bias.hpp
index 33e89830c2..ae7c2832dd 100644
--- a/src/mlpack/methods/ann/layer/linear_no_bias.hpp
+++ b/src/mlpack/methods/ann/layer/linear_no_bias.hpp
@@ -14,7 +14,7 @@
 #define MLPACK_METHODS_ANN_LAYER_LINEAR_NO_BIAS_HPP
 
 #include <mlpack/prereqs.hpp>
-#include <mlpack/methods/ann/regularizer/regularizer.hpp>
+#include <mlpack/methods/ann/regularizer/no_regularizer.hpp>
 
 #include "layer_types.hpp"
 
diff --git a/src/mlpack/methods/ann/regularizer/no_regularizer.hpp b/src/mlpack/methods/ann/regularizer/no_regularizer.hpp
index f411816f18..b480567ab0 100644
--- a/src/mlpack/methods/ann/regularizer/no_regularizer.hpp
+++ b/src/mlpack/methods/ann/regularizer/no_regularizer.hpp
@@ -9,8 +9,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_METHODS_ANN_NOREGULARIZER_HPP
-#define MLPACK_METHODS_ANN_NOREGULARIZER_HPP
+#ifndef MLPACK_METHODS_ANN_NO_REGULARIZER_HPP
+#define MLPACK_METHODS_ANN_NO_REGULARIZER_HPP
 
 #include <mlpack/prereqs.hpp>
 
diff --git a/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp
index 83dde5563a..9c0d243a72 100644
--- a/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp
+++ b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp
@@ -9,8 +9,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_METHODS_ANN_LREGULARIZER_HPP
-#define MLPACK_METHODS_ANN_LREGULARIZER_HPP
+#ifndef MLPACK_METHODS_ANN_ORTHOGONAL_REGULARIZER_HPP
+#define MLPACK_METHODS_ANN_ORTHOGONAL_REGULARIZER_HPP
 
 #include <mlpack/prereqs.hpp>
 
diff --git a/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp
index 2e41aa9c5d..a40ccf29bf 100644
--- a/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp
+++ b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp
@@ -9,8 +9,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_METHODS_ANN_LREGULARIZER_IMPL_HPP
-#define MLPACK_METHODS_ANN_LREGULARIZER_IMPL_HPP
+#ifndef MLPACK_METHODS_ANN_ORTHOGONAL_REGULARIZER_IMPL_HPP
+#define MLPACK_METHODS_ANN_ORTHOGONAL_REGULARIZER_IMPL_HPP
 
 // In case it hasn't been included.
 #include "orthogonal_regularizer.hpp"
@@ -49,12 +49,11 @@ void OrthogonalRegularizer::Evaluate(const MatType& weight, MatType& gradient)
     }
   }
 
-  gradient += arma::vectorise(grad) * factor;
+  gradient = arma::vectorise(grad) * factor;
 }
 
-template<int Power>
 template<typename Archive>
-void OrthogonalRegularizer<Power>::serialize(
+void OrthogonalRegularizer::serialize(
     Archive& ar, const unsigned int /* version */)
 {
   ar & BOOST_SERIALIZATION_NVP(factor);
diff --git a/src/mlpack/methods/ann/regularizer/regularizer.hpp b/src/mlpack/methods/ann/regularizer/regularizer.hpp
index 035350563b..c5dd506170 100644
--- a/src/mlpack/methods/ann/regularizer/regularizer.hpp
+++ b/src/mlpack/methods/ann/regularizer/regularizer.hpp
@@ -9,8 +9,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_METHODS_ANN_REGULARIZER_LAYER_HPP
-#define MLPACK_METHODS_ANN_REGULARIZER_LAYER_HPP
+#ifndef MLPACK_METHODS_ANN_REGULARIZER_HPP
+#define MLPACK_METHODS_ANN_REGULARIZER_HPP
 
 #include "lregularizer.hpp"
 #include "orthogonal_regularizer.hpp"
diff --git a/src/mlpack/tests/ann_regularizer_test.cpp b/src/mlpack/tests/ann_regularizer_test.cpp
index 9d60f9deea..4de678f741 100644
--- a/src/mlpack/tests/ann_regularizer_test.cpp
+++ b/src/mlpack/tests/ann_regularizer_test.cpp
@@ -17,7 +17,6 @@
 #include <mlpack/methods/ann/regularizer/regularizer.hpp>
 
 #include <boost/test/unit_test.hpp>
-#include "test_tools.hpp"
 #include "ann_test_tools.hpp"
 #include "serialization.hpp"
 
@@ -84,4 +83,36 @@ BOOST_AUTO_TEST_CASE(GradientL2RegularizerTest)
   BOOST_REQUIRE_LE(CheckRegularizerGradient(function), 1e-4);
 }
 
+BOOST_AUTO_TEST_CASE(GradientOrthogonalRegularizerTest)
+{
+  // Add function gradient instantiation.
+  struct GradientFunction
+  {
+    GradientFunction() :
+      factor(0.6),
+      reg(factor)
+    {
+      // Nothing to do here.
+    }
+
+    double Output(const arma::mat& weight, size_t /* i */, size_t /* j */)
+    {
+      arma::mat I;
+      I.eye(weight.n_rows, weight.n_rows);
+      arma::mat x = arma::abs(weight * weight.t() - I) * factor;
+      return arma::accu(x);
+    }
+
+    void Gradient(arma::mat& weight, arma::mat& gradient)
+    {
+      reg.Evaluate(weight, gradient);
+    }
+
+    double factor;
+    OrthogonalRegularizer reg;
+  } function;
+
+  BOOST_REQUIRE_LE(CheckRegularizerGradient(function), 1e-4);
+}
+
 BOOST_AUTO_TEST_SUITE_END();
diff --git a/src/mlpack/tests/ann_test_tools.hpp b/src/mlpack/tests/ann_test_tools.hpp
index 2c8a9dbf86..214550dea4 100644
--- a/src/mlpack/tests/ann_test_tools.hpp
+++ b/src/mlpack/tests/ann_test_tools.hpp
@@ -173,7 +173,7 @@ double CheckRegularizerGradient(FunctionType& function, const double eps = 1e-7)
 {
   // Get gradients for the current parameters.
   arma::mat orgGradient, estGradient;
-  arma::mat weight = arma::randu(10, 15);
+  arma::mat weight = arma::randu(10, 10);
   function.Gradient(weight, orgGradient);
 
   estGradient = arma::zeros(weight.n_rows, weight.n_cols);
@@ -183,13 +183,15 @@ double CheckRegularizerGradient(FunctionType& function, const double eps = 1e-7)
   {
     for (size_t j = 0; j < weight.n_cols; ++j)
     {
+      double tmp = weight(i, j);
+
       weight(i, j) += eps;
       double costPlus = function.Output(weight, i, j);
       weight(i, j) -= (2 * eps);
       double costMinus = function.Output(weight, i, j);
 
       // Restore the weight value.
-      weight(i, j) += eps;
+      weight(i, j) = tmp;
       estGradient(i, j) = (costPlus - costMinus) / (2 * eps);
     }
   }

From 6a75208f11f3d33a3b178ac4420217a8240ad325 Mon Sep 17 00:00:00 2001
From: Saksham Bansal <7020962+saksham189@users.noreply.github.com>
Date: Sat, 10 Aug 2019 03:51:15 +0530
Subject: [PATCH 09/11] Style fix

---
 src/mlpack/tests/ann_regularizer_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/tests/ann_regularizer_test.cpp b/src/mlpack/tests/ann_regularizer_test.cpp
index 4de678f741..4ae77143c8 100644
--- a/src/mlpack/tests/ann_regularizer_test.cpp
+++ b/src/mlpack/tests/ann_regularizer_test.cpp
@@ -2,7 +2,7 @@
  * @file ann_regularizer_test.cpp
  * @author Saksham Bansal
  *
- * Tests the ann regularizer modules.
+ * Tests the ANN regularizer modules.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the

From 10c1695004191487adfcd1d19310d6ddc420b382 Mon Sep 17 00:00:00 2001
From: Saksham Bansal <7020962+saksham189@users.noreply.github.com>
Date: Sat, 10 Aug 2019 03:56:26 +0530
Subject: [PATCH 10/11] Add gradient

---
 src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp    | 6 +++---
 .../methods/ann/regularizer/orthogonal_regularizer_impl.hpp | 2 +-
 src/mlpack/tests/ann_test_tools.hpp                         | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp b/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp
index 6c07484ace..469fc1311c 100644
--- a/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp
+++ b/src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp
@@ -30,7 +30,7 @@ template<int Power>
 template<typename MatType>
 void LRegularizer<Power>::Evaluate(const MatType& weight, MatType& gradient)
 {
-  gradient = arma::vectorise(arma::pow(weight, Power - 1) * Power * factor);
+  gradient += arma::vectorise(arma::pow(weight, Power - 1) * Power * factor);
 }
 
 // L1-Regularizer specializations.
@@ -38,7 +38,7 @@ template<>
 template<typename MatType>
 void LRegularizer<1>::Evaluate(const MatType& weight, MatType& gradient)
 {
-  gradient = arma::vectorise(factor * weight / arma::abs(weight));
+  gradient += arma::vectorise(factor * weight / arma::abs(weight));
 }
 
 // L2-Regularizer specializations.
@@ -46,7 +46,7 @@ template<>
 template<typename MatType>
 void LRegularizer<2>::Evaluate(const MatType& weight, MatType& gradient)
 {
-  gradient = arma::vectorise(2 * factor * weight);
+  gradient += arma::vectorise(2 * factor * weight);
 }
 
 template<int Power>
diff --git a/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp
index a40ccf29bf..fd529cd7bd 100644
--- a/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp
+++ b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp
@@ -49,7 +49,7 @@ void OrthogonalRegularizer::Evaluate(const MatType& weight, MatType& gradient)
     }
   }
 
-  gradient = arma::vectorise(grad) * factor;
+  gradient += arma::vectorise(grad) * factor;
 }
 
 template<typename Archive>
diff --git a/src/mlpack/tests/ann_test_tools.hpp b/src/mlpack/tests/ann_test_tools.hpp
index 214550dea4..a68d300479 100644
--- a/src/mlpack/tests/ann_test_tools.hpp
+++ b/src/mlpack/tests/ann_test_tools.hpp
@@ -172,11 +172,11 @@ template<class FunctionType>
 double CheckRegularizerGradient(FunctionType& function, const double eps = 1e-7)
 {
   // Get gradients for the current parameters.
-  arma::mat orgGradient, estGradient;
   arma::mat weight = arma::randu(10, 10);
+  arma::mat orgGradient = arma::zeros(10, 10);
   function.Gradient(weight, orgGradient);
 
-  estGradient = arma::zeros(weight.n_rows, weight.n_cols);
+  arma::mat estGradient = arma::zeros(weight.n_rows, weight.n_cols);
 
   // Compute numeric approximations to gradient.
   for (size_t i = 0; i < weight.n_rows; ++i)

From 1c3ebc380f9da77075cac5cef25c426bd679b02d Mon Sep 17 00:00:00 2001
From: Saksham Bansal <7020962+saksham189@users.noreply.github.com>
Date: Sat, 10 Aug 2019 16:42:30 +0530
Subject: [PATCH 11/11] Style fix

---
 .../regularizer/orthogonal_regularizer_impl.hpp |  7 ++++---
 src/mlpack/tests/ann_regularizer_test.cpp       | 17 ++++++++---------
 src/mlpack/tests/ann_test_tools.hpp             |  2 +-
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp
index fd529cd7bd..d1f3411e7e 100644
--- a/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp
+++ b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp
@@ -27,7 +27,7 @@ OrthogonalRegularizer::OrthogonalRegularizer(double factor) :
 template<typename MatType>
 void OrthogonalRegularizer::Evaluate(const MatType& weight, MatType& gradient)
 {
-  arma::mat grad = arma::mat(arma::size(weight), arma::fill::zeros);
+  arma::mat grad = arma::zeros(arma::size(weight));
 
   for (size_t i = 0; i < weight.n_rows; i++)
   {
@@ -35,14 +35,15 @@ void OrthogonalRegularizer::Evaluate(const MatType& weight, MatType& gradient)
     {
       if (i == j)
       {
-        int s =
+        double s =
             arma::as_scalar(
             arma::sign((weight.row(i) * weight.row(i).t()) - 1));
         grad.row(i) += 2 * s * weight.row(i);
       }
       else
       {
-        int s = arma::as_scalar(arma::sign(weight.row(i) * weight.row(j).t()));
+        double s = arma::as_scalar(
+            arma::sign(weight.row(i) * weight.row(j).t()));
         grad.row(i) += s * weight.row(j);
         grad.row(j) += s * weight.row(i);
       }
diff --git a/src/mlpack/tests/ann_regularizer_test.cpp b/src/mlpack/tests/ann_regularizer_test.cpp
index 4ae77143c8..ba7f3b58ee 100644
--- a/src/mlpack/tests/ann_regularizer_test.cpp
+++ b/src/mlpack/tests/ann_regularizer_test.cpp
@@ -39,7 +39,7 @@ BOOST_AUTO_TEST_CASE(GradientL1RegularizerTest)
 
     double Output(const arma::mat& weight, size_t i, size_t j)
     {
-     return std::abs(weight(i, j)) * factor;
+      return std::abs(weight(i, j)) * factor;
     }
 
     void Gradient(arma::mat& weight, arma::mat& gradient)
@@ -60,15 +60,15 @@ BOOST_AUTO_TEST_CASE(GradientL2RegularizerTest)
   struct GradientFunction
   {
     GradientFunction() :
-      factor(0.6),
-      reg(factor)
+        factor(0.6),
+        reg(factor)
     {
       // Nothing to do here.
     }
 
     double Output(const arma::mat& weight, size_t i, size_t j)
     {
-     return weight(i, j) * weight(i, j) * factor;
+      return weight(i, j) * weight(i, j) * factor;
     }
 
     void Gradient(arma::mat& weight, arma::mat& gradient)
@@ -89,17 +89,16 @@ BOOST_AUTO_TEST_CASE(GradientOrthogonalRegularizerTest)
   struct GradientFunction
   {
     GradientFunction() :
-      factor(0.6),
-      reg(factor)
+        factor(0.6),
+        reg(factor)
     {
       // Nothing to do here.
     }
 
     double Output(const arma::mat& weight, size_t /* i */, size_t /* j */)
     {
-      arma::mat I;
-      I.eye(weight.n_rows, weight.n_rows);
-      arma::mat x = arma::abs(weight * weight.t() - I) * factor;
+      arma::mat x = arma::abs(weight * weight.t() -
+          arma::eye<arma::mat>(weight.n_rows, weight.n_cols)) * factor;
       return arma::accu(x);
     }
 
diff --git a/src/mlpack/tests/ann_test_tools.hpp b/src/mlpack/tests/ann_test_tools.hpp
index a68d300479..be789befd5 100644
--- a/src/mlpack/tests/ann_test_tools.hpp
+++ b/src/mlpack/tests/ann_test_tools.hpp
@@ -173,7 +173,7 @@ double CheckRegularizerGradient(FunctionType& function, const double eps = 1e-7)
 {
   // Get gradients for the current parameters.
   arma::mat weight = arma::randu(10, 10);
-  arma::mat orgGradient = arma::zeros(10, 10);
+  arma::mat orgGradient = arma::zeros(10 * 10, 1);
   function.Gradient(weight, orgGradient);
 
   arma::mat estGradient = arma::zeros(weight.n_rows, weight.n_cols);