Merge pull request #1932 from saksham189/reg

Add Regularizers

src/mlpack/methods/ann/CMakeLists.txt

@@ -18,6 +18,7 @@ add_subdirectory(convolution_rules)
 add_subdirectory(gan)
 add_subdirectory(rbm)
 add_subdirectory(augmented)
+add_subdirectory(regularizer)
 
 # Add directory name to sources.
 set(DIR_SRCS)

src/mlpack/methods/ann/layer/layer_types.hpp

@@ -47,6 +47,9 @@
 #include <mlpack/methods/ann/convolution_rules/naive_convolution.hpp>
 #include <mlpack/methods/ann/convolution_rules/fft_convolution.hpp>
 
+// Regularizers.
+#include <mlpack/methods/ann/regularizer/no_regularizer.hpp>
+
 // Loss function modules.
 #include <mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp>
 
@@ -57,14 +60,22 @@ template<typename InputDataType, typename OutputDataType> class BatchNorm;
 template<typename InputDataType, typename OutputDataType> class DropConnect;
 template<typename InputDataType, typename OutputDataType> class Glimpse;
 template<typename InputDataType, typename OutputDataType> class LayerNorm;
-template<typename InputDataType, typename OutputDataType> class Linear;
-template<typename InputDataType, typename OutputDataType> class LinearNoBias;
 template<typename InputDataType, typename OutputDataType> class LSTM;
 template<typename InputDataType, typename OutputDataType> class GRU;
 template<typename InputDataType, typename OutputDataType> class FastLSTM;
 template<typename InputDataType, typename OutputDataType> class VRClassReward;
 template<typename InputDataType, typename OutputDataType> class Concatenate;
 
+template<typename InputDataType,
+         typename OutputDataType,
+         typename RegularizerType>
+class Linear;
+
+template<typename InputDataType,
+         typename OutputDataType,
+         typename RegularizerType>
+class LinearNoBias;
+
 template<typename InputDataType,
          typename OutputDataType
 >
@@ -190,8 +201,8 @@ using LayerTypes = boost::variant<
     LayerNorm<arma::mat, arma::mat>*,
     LeakyReLU<arma::mat, arma::mat>*,
     CReLU<arma::mat, arma::mat>*,
-    Linear<arma::mat, arma::mat>*,
-    LinearNoBias<arma::mat, arma::mat>*,
+    Linear<arma::mat, arma::mat, NoRegularizer>*,
+    LinearNoBias<arma::mat, arma::mat, NoRegularizer>*,
     LogSoftMax<arma::mat, arma::mat>*,
     Lookup<arma::mat, arma::mat>*,
     LSTM<arma::mat, arma::mat>*,

src/mlpack/methods/ann/layer/linear.hpp

@@ -14,6 +14,7 @@
 #define MLPACK_METHODS_ANN_LAYER_LINEAR_HPP
 
 #include <mlpack/prereqs.hpp>
+#include <mlpack/methods/ann/regularizer/no_regularizer.hpp>
 
 #include "layer_types.hpp"
 
@@ -31,7 +32,8 @@ namespace ann /** Artificial Neural Network. */ {
  */
 template <
     typename InputDataType = arma::mat,
-    typename OutputDataType = arma::mat
+    typename OutputDataType = arma::mat,
+    typename RegularizerType = NoRegularizer
 >
 class Linear
 {
@@ -45,7 +47,9 @@ class Linear
    * @param inSize The number of input units.
    * @param outSize The number of output units.
    */
-  Linear(const size_t inSize, const size_t outSize);
+  Linear(const size_t inSize,
+         const size_t outSize,
+         RegularizerType regularizer = RegularizerType());
 
   /*
    * Reset the layer parameter.
@@ -146,6 +150,9 @@ class Linear
 
   //! Locally-stored output parameter object.
   OutputDataType outputParameter;
+
+  //! Locally-stored regularizer object.
+  RegularizerType regularizer;
 }; // class Linear
 
 } // namespace ann

src/mlpack/methods/ann/layer/linear_impl.hpp

@@ -19,50 +19,60 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template<typename InputDataType, typename OutputDataType>
-Linear<InputDataType, OutputDataType>::Linear()
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
+Linear<InputDataType, OutputDataType, RegularizerType>::Linear() :
+    inSize(0),
+    outSize(0)
 {
   // Nothing to do here.
 }
 
-template <typename InputDataType, typename OutputDataType>
-Linear<InputDataType, OutputDataType>::Linear(
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
+Linear<InputDataType, OutputDataType, RegularizerType>::Linear(
     const size_t inSize,
-    const size_t outSize) :
+    const size_t outSize,
+    RegularizerType regularizer) :
     inSize(inSize),
-    outSize(outSize)
+    outSize(outSize),
+    regularizer(regularizer)
 {
   weights.set_size(outSize * inSize + outSize, 1);
 }
 
-template<typename InputDataType, typename OutputDataType>
-void Linear<InputDataType, OutputDataType>::Reset()
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
+void Linear<InputDataType, OutputDataType, RegularizerType>::Reset()
 {
   weight = arma::mat(weights.memptr(), outSize, inSize, false, false);
   bias = arma::mat(weights.memptr() + weight.n_elem,
       outSize, 1, false, false);
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 template<typename eT>
-void Linear<InputDataType, OutputDataType>::Forward(
+void Linear<InputDataType, OutputDataType, RegularizerType>::Forward(
     const arma::Mat<eT>&& input, arma::Mat<eT>&& output)
 {
   output = weight * input;
   output.each_col() += bias;
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 template<typename eT>
-void Linear<InputDataType, OutputDataType>::Backward(
+void Linear<InputDataType, OutputDataType, RegularizerType>::Backward(
     const arma::Mat<eT>&& /* input */, arma::Mat<eT>&& gy, arma::Mat<eT>&& g)
 {
   g = weight.t() * gy;
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 template<typename eT>
-void Linear<InputDataType, OutputDataType>::Gradient(
+void Linear<InputDataType, OutputDataType, RegularizerType>::Gradient(
     const arma::Mat<eT>&& input,
     arma::Mat<eT>&& error,
     arma::Mat<eT>&& gradient)
@@ -71,11 +81,13 @@ void Linear<InputDataType, OutputDataType>::Gradient(
       error * input.t());
   gradient.submat(weight.n_elem, 0, gradient.n_elem - 1, 0) =
       arma::sum(error, 1);
+  regularizer.Evaluate(weights, gradient);
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 template<typename Archive>
-void Linear<InputDataType, OutputDataType>::serialize(
+void Linear<InputDataType, OutputDataType, RegularizerType>::serialize(
     Archive& ar, const unsigned int /* version */)
 {
   ar & BOOST_SERIALIZATION_NVP(inSize);

src/mlpack/methods/ann/layer/linear_no_bias.hpp

@@ -14,6 +14,7 @@
 #define MLPACK_METHODS_ANN_LAYER_LINEAR_NO_BIAS_HPP
 
 #include <mlpack/prereqs.hpp>
+#include <mlpack/methods/ann/regularizer/no_regularizer.hpp>
 
 #include "layer_types.hpp"
 
@@ -31,7 +32,8 @@ namespace ann /** Artificial Neural Network. */ {
  */
 template <
     typename InputDataType = arma::mat,
-    typename OutputDataType = arma::mat
+    typename OutputDataType = arma::mat,
+    typename RegularizerType = NoRegularizer
 >
 class LinearNoBias
 {
@@ -44,7 +46,9 @@ class LinearNoBias
    * @param inSize The number of input units.
    * @param outSize The number of output units.
    */
-  LinearNoBias(const size_t inSize, const size_t outSize);
+  LinearNoBias(const size_t inSize,
+               const size_t outSize,
+               RegularizerType regularizer = RegularizerType());
 
   /*
    * Reset the layer parameter.
@@ -142,6 +146,9 @@ class LinearNoBias
 
   //! Locally-stored output parameter object.
   OutputDataType outputParameter;
+
+  //! Locally-stored regularizer object.
+  RegularizerType regularizer;
 }; // class LinearNoBias
 
 } // namespace ann

src/mlpack/methods/ann/layer/linear_no_bias_impl.hpp

@@ -19,57 +19,70 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template<typename InputDataType, typename OutputDataType>
-LinearNoBias<InputDataType, OutputDataType>::LinearNoBias()
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
+LinearNoBias<InputDataType, OutputDataType, RegularizerType>::LinearNoBias() :
+    inSize(0),
+    outSize(0)
 {
   // Nothing to do here.
 }
 
-template<typename InputDataType, typename OutputDataType>
-LinearNoBias<InputDataType, OutputDataType>::LinearNoBias(
-    const size_t inSize, const size_t outSize) :
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
+LinearNoBias<InputDataType, OutputDataType, RegularizerType>::LinearNoBias(
+    const size_t inSize,
+    const size_t outSize,
+    RegularizerType regularizer) :
     inSize(inSize),
-    outSize(outSize)
+    outSize(outSize),
+    regularizer(regularizer)
 {
   weights.set_size(outSize * inSize, 1);
 }
 
-template <typename InputDataType, typename OutputDataType>
-void LinearNoBias<InputDataType, OutputDataType>::Reset()
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
+void LinearNoBias<InputDataType, OutputDataType, RegularizerType>::Reset()
 {
   weight = arma::mat(weights.memptr(), outSize, inSize, false, false);
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 template<typename eT>
-void LinearNoBias<InputDataType, OutputDataType>::Forward(
+void LinearNoBias<InputDataType, OutputDataType, RegularizerType>::Forward(
     const arma::Mat<eT>&& input, arma::Mat<eT>&& output)
 {
   output = weight * input;
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 template<typename eT>
-void LinearNoBias<InputDataType, OutputDataType>::Backward(
+void LinearNoBias<InputDataType, OutputDataType, RegularizerType>::Backward(
     const arma::Mat<eT>&& /* input */, arma::Mat<eT>&& gy, arma::Mat<eT>&& g)
 {
   g = weight.t() * gy;
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 template<typename eT>
-void LinearNoBias<InputDataType, OutputDataType>::Gradient(
+void LinearNoBias<InputDataType, OutputDataType, RegularizerType>::Gradient(
     const arma::Mat<eT>&& input,
     arma::Mat<eT>&& error,
     arma::Mat<eT>&& gradient)
 {
   gradient.submat(0, 0, weight.n_elem - 1, 0) = arma::vectorise(
       error * input.t());
+  regularizer.Evaluate(weights, gradient);
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 template<typename Archive>
-void LinearNoBias<InputDataType, OutputDataType>::serialize(
+void LinearNoBias<InputDataType, OutputDataType, RegularizerType>::serialize(
     Archive& ar, const unsigned int /* version */)
 {
   ar & BOOST_SERIALIZATION_NVP(inSize);

src/mlpack/methods/ann/regularizer/CMakeLists.txt

@@ -0,0 +1,18 @@
+# Define the files we need to compile.
+# Anything not in this list will not be compiled into mlpack.
+set(SOURCES
+  lregularizer.hpp
+  lregularizer_impl.hpp
+  orthogonal_regularizer.hpp
+  orthogonal_regularizer_impl.hpp
+  no_regularizer.hpp
+)
+
+# Add directory name to sources.
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+    set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+# Append sources (with directory name) to list of all mlpack sources (used at
+# the parent scope).
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)

src/mlpack/methods/ann/regularizer/lregularizer.hpp

@@ -0,0 +1,75 @@
+/**
+ * @file lregularizer.hpp
+ * @author Saksham Bansal
+ *
+ * Generalized L-regularizer, allowing both l1 and l2 regularization methods.
+ * This also gives several convenience typedefs for commonly used L-regularizers.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LREGULARIZER_HPP
+#define MLPACK_METHODS_ANN_LREGULARIZER_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann {
+
+/**
+ * The L_p regularizer for arbitrary integer p.
+ *
+ * @tparam Power Power of regularizer; i.e. Power = 1 gives the L1-regularization.
+ */
+template<int TPower>
+class LRegularizer
+{
+ public:
+  /**
+   * Create the regularizer object.
+   *
+   * @param factor The factor for regularization.
+   */
+  LRegularizer(double factor = 1.0);
+
+  /**
+   * Calculate the gradient for regularization.
+   *
+   * @tparam MatType Type of weight matrix.
+   * @param weight The weight matrix to be regularized.
+   * @param gradient The calculated gradient.
+   */
+  template<typename MatType>
+  void Evaluate(const MatType& weight, MatType& gradient);
+
+  //! Serialize the regularizer (nothing to do).
+  template<typename Archive>
+  void serialize(Archive& ar, const unsigned int /* version */);
+
+  //! The power of the regularizer.
+  static const int Power = TPower;
+
+  //! The constant for the regularization
+  double factor;
+};
+
+// Convenience typedefs.
+/**
+ * The L1 Regularizer.
+ */
+typedef LRegularizer<1> L1Regularizer;
+
+/**
+ * The L2 Regularizer.
+ */
+typedef LRegularizer<2> L2Regularizer;
+
+} // namespace ann
+} // namespace mlpack
+
+// Include implementation.
+#include "lregularizer_impl.hpp"
+
+#endif

src/mlpack/methods/ann/regularizer/lregularizer_impl.hpp

@@ -0,0 +1,63 @@
+/**
+ * @file lregularization_impl.hpp
+ * @author Saksham Bansal
+ *
+ * Implementation of template specializations of LRegularizer class.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LREGULARIZER_IMPL_HPP
+#define MLPACK_METHODS_ANN_LREGULARIZER_IMPL_HPP
+
+// In case it hasn't been included.
+#include "lregularizer.hpp"
+
+namespace mlpack {
+namespace ann {
+
+template<int Power>
+LRegularizer<Power>::LRegularizer(double factor) :
+    factor(factor)
+{
+  // Nothing to do here
+}
+
+// Unspecialized implementation. This should almost never be used...
+template<int Power>
+template<typename MatType>
+void LRegularizer<Power>::Evaluate(const MatType& weight, MatType& gradient)
+{
+  gradient += arma::vectorise(arma::pow(weight, Power - 1) * Power * factor);
+}
+
+// L1-Regularizer specializations.
+template<>
+template<typename MatType>
+void LRegularizer<1>::Evaluate(const MatType& weight, MatType& gradient)
+{
+  gradient += arma::vectorise(factor * weight / arma::abs(weight));
+}
+
+// L2-Regularizer specializations.
+template<>
+template<typename MatType>
+void LRegularizer<2>::Evaluate(const MatType& weight, MatType& gradient)
+{
+  gradient += arma::vectorise(2 * factor * weight);
+}
+
+template<int Power>
+template<typename Archive>
+void LRegularizer<Power>::serialize(
+    Archive& ar, const unsigned int /* version */)
+{
+  ar & BOOST_SERIALIZATION_NVP(factor);
+}
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/regularizer/no_regularizer.hpp

@@ -0,0 +1,52 @@
+/**
+ * @file no_regularizer.hpp
+ * @author Saksham Bansal
+ *
+ * Definition of the NoRegularizer class.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_NO_REGULARIZER_HPP
+#define MLPACK_METHODS_ANN_NO_REGULARIZER_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann {
+
+/**
+ * Implementation of the NoRegularizer. This does not add any
+ * regularization to the weights.
+ */
+class NoRegularizer
+{
+ public:
+  /**
+   * Create the regularizer object.
+   */
+  NoRegularizer()
+  {
+    // Nothing to do here.
+  };
+
+  /**
+   * Calculate the gradient for regularization.
+   *
+   * @tparam MatType Type of weight matrix.
+   * @param weight The weight matrix to be regularized.
+   * @param gradient The calculated gradient.
+   */
+  template<typename MatType>
+  void Evaluate(const MatType& /* weight */, MatType& /* gradient */)
+  {
+    // Nothing to do here.
+  }
+};
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp

@@ -0,0 +1,72 @@
+/**
+ * @file orthogonal_regularizer.hpp
+ * @author Saksham Bansal
+ *
+ * Definition of the OrthogonalRegularizer class.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_ORTHOGONAL_REGULARIZER_HPP
+#define MLPACK_METHODS_ANN_ORTHOGONAL_REGULARIZER_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann {
+
+/**
+ * Implementation of the OrthogonalRegularizer. Orthogonality of weights is a
+ * desirable property because multiplication by an orthogonal matrix leaves
+ * the norm of the matrix unchanged. The orthogonal regularization technique
+ * encourages weights to be orthogonal.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @inproceedings{WanICML2013,
+ *   title={Neural Photo Editing with Introspective Adversarial Networks},
+ *   booktitle = {5th International Conference on Learning Representations
+ *                (ICLR - 17)},
+ *   author = {Andrew Brock and Theodore Lim and J.M. Ritchie and Nick Weston},
+ *   year = {2017}
+ * }
+ * @endcode
+ */
+class OrthogonalRegularizer
+{
+ public:
+  /**
+   * Create the regularizer object.
+   *
+   * @param factor The factor for regularization.
+   */
+  OrthogonalRegularizer(double factor = 1.0);
+
+  /**
+   * Calculate the gradient for regularization.
+   *
+   * @tparam MatType Type of weight matrix.
+   * @param weight The weight matrix to be regularized.
+   * @param gradient The calculated gradient.
+   */
+  template<typename MatType>
+  void Evaluate(const MatType& weight, MatType& gradient);
+
+  //! Serialize the regularizer (nothing to do).
+  template<typename Archive>
+  void serialize(Archive& ar, const unsigned int /* version */);
+
+  //! The constant for the regularization
+  double factor;
+};
+
+} // namespace ann
+} // namespace mlpack
+
+// Include implementation.
+#include "orthogonal_regularizer_impl.hpp"
+
+#endif

src/mlpack/methods/ann/regularizer/orthogonal_regularizer_impl.hpp

@@ -0,0 +1,66 @@
+/**
+ * @file orthogonal_regularizer_impl.hpp
+ * @author Saksham Bansal
+ *
+ * Implementation of OrthogonalRegularizer class.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_ORTHOGONAL_REGULARIZER_IMPL_HPP
+#define MLPACK_METHODS_ANN_ORTHOGONAL_REGULARIZER_IMPL_HPP
+
+// In case it hasn't been included.
+#include "orthogonal_regularizer.hpp"
+
+namespace mlpack {
+namespace ann {
+
+OrthogonalRegularizer::OrthogonalRegularizer(double factor) :
+    factor(factor)
+{
+  // Nothing to do here.
+}
+
+template<typename MatType>
+void OrthogonalRegularizer::Evaluate(const MatType& weight, MatType& gradient)
+{
+  arma::mat grad = arma::zeros(arma::size(weight));
+
+  for (size_t i = 0; i < weight.n_rows; i++)
+  {
+    for (size_t j = 0; j < weight.n_rows; j++)
+    {
+      if (i == j)
+      {
+        double s =
+            arma::as_scalar(
+            arma::sign((weight.row(i) * weight.row(i).t()) - 1));
+        grad.row(i) += 2 * s * weight.row(i);
+      }
+      else
+      {
+        double s = arma::as_scalar(
+            arma::sign(weight.row(i) * weight.row(j).t()));
+        grad.row(i) += s * weight.row(j);
+        grad.row(j) += s * weight.row(i);
+      }
+    }
+  }
+
+  gradient += arma::vectorise(grad) * factor;
+}
+
+template<typename Archive>
+void OrthogonalRegularizer::serialize(
+    Archive& ar, const unsigned int /* version */)
+{
+  ar & BOOST_SERIALIZATION_NVP(factor);
+}
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/regularizer/regularizer.hpp

@@ -0,0 +1,19 @@
+/**
+ * @file regularizer.hpp
+ * @author Saksham Bansal
+ *
+ * This includes various regularizers to construct a model.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_REGULARIZER_HPP
+#define MLPACK_METHODS_ANN_REGULARIZER_HPP
+
+#include "lregularizer.hpp"
+#include "orthogonal_regularizer.hpp"
+#include "no_regularizer.hpp"
+
+#endif

src/mlpack/tests/CMakeLists.txt

@@ -6,6 +6,7 @@ add_executable(mlpack_test
   aknn_test.cpp
   ann_dist_test.cpp
   ann_layer_test.cpp
+  ann_regularizer_test.cpp
   ann_test_tools.hpp
   arma_extend_test.cpp
   armadillo_svd_test.cpp

src/mlpack/tests/ann_regularizer_test.cpp

@@ -0,0 +1,117 @@
+/**
+ * @file ann_regularizer_test.cpp
+ * @author Saksham Bansal
+ *
+ * Tests the ANN regularizer modules.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include <mlpack/core.hpp>
+
+#include <mlpack/methods/ann/layer/layer.hpp>
+#include <mlpack/methods/ann/layer/layer_types.hpp>
+#include <mlpack/methods/ann/init_rules/random_init.hpp>
+#include <mlpack/methods/ann/regularizer/regularizer.hpp>
+
+#include <boost/test/unit_test.hpp>
+#include "ann_test_tools.hpp"
+#include "serialization.hpp"
+
+using namespace mlpack;
+using namespace mlpack::ann;
+
+BOOST_AUTO_TEST_SUITE(ANNRegularizerTest);
+
+BOOST_AUTO_TEST_CASE(GradientL1RegularizerTest)
+{
+  // Add function gradient instantiation.
+  struct GradientFunction
+  {
+    GradientFunction() :
+      factor(0.6),
+      reg(factor)
+    {
+      // Nothing to do here.
+    }
+
+    double Output(const arma::mat& weight, size_t i, size_t j)
+    {
+      return std::abs(weight(i, j)) * factor;
+    }
+
+    void Gradient(arma::mat& weight, arma::mat& gradient)
+    {
+      reg.Evaluate(weight, gradient);
+    }
+
+    double factor;
+    L1Regularizer reg;
+  } function;
+
+  BOOST_REQUIRE_LE(CheckRegularizerGradient(function), 1e-4);
+}
+
+BOOST_AUTO_TEST_CASE(GradientL2RegularizerTest)
+{
+  // Add function gradient instantiation.
+  struct GradientFunction
+  {
+    GradientFunction() :
+        factor(0.6),
+        reg(factor)
+    {
+      // Nothing to do here.
+    }
+
+    double Output(const arma::mat& weight, size_t i, size_t j)
+    {
+      return weight(i, j) * weight(i, j) * factor;
+    }
+
+    void Gradient(arma::mat& weight, arma::mat& gradient)
+    {
+      reg.Evaluate(weight, gradient);
+    }
+
+    double factor;
+    L2Regularizer reg;
+  } function;
+
+  BOOST_REQUIRE_LE(CheckRegularizerGradient(function), 1e-4);
+}
+
+BOOST_AUTO_TEST_CASE(GradientOrthogonalRegularizerTest)
+{
+  // Add function gradient instantiation.
+  struct GradientFunction
+  {
+    GradientFunction() :
+        factor(0.6),
+        reg(factor)
+    {
+      // Nothing to do here.
+    }
+
+    double Output(const arma::mat& weight, size_t /* i */, size_t /* j */)
+    {
+      arma::mat x = arma::abs(weight * weight.t() -
+          arma::eye<arma::mat>(weight.n_rows, weight.n_cols)) * factor;
+      return arma::accu(x);
+    }
+
+    void Gradient(arma::mat& weight, arma::mat& gradient)
+    {
+      reg.Evaluate(weight, gradient);
+    }
+
+    double factor;
+    OrthogonalRegularizer reg;
+  } function;
+
+  BOOST_REQUIRE_LE(CheckRegularizerGradient(function), 1e-4);
+}
+
+BOOST_AUTO_TEST_SUITE_END();

src/mlpack/tests/ann_test_tools.hpp

@@ -167,4 +167,39 @@ double CheckGradient(FunctionType& function, const double eps = 1e-7)
       arma::norm(orgGradient + estGradient);
 }
 
+// Simple numerical gradient checker for regularizers.
+template<class FunctionType>
+double CheckRegularizerGradient(FunctionType& function, const double eps = 1e-7)
+{
+  // Get gradients for the current parameters.
+  arma::mat weight = arma::randu(10, 10);
+  arma::mat orgGradient = arma::zeros(10 * 10, 1);
+  function.Gradient(weight, orgGradient);
+
+  arma::mat estGradient = arma::zeros(weight.n_rows, weight.n_cols);
+
+  // Compute numeric approximations to gradient.
+  for (size_t i = 0; i < weight.n_rows; ++i)
+  {
+    for (size_t j = 0; j < weight.n_cols; ++j)
+    {
+      double tmp = weight(i, j);
+
+      weight(i, j) += eps;
+      double costPlus = function.Output(weight, i, j);
+      weight(i, j) -= (2 * eps);
+      double costMinus = function.Output(weight, i, j);
+
+      // Restore the weight value.
+      weight(i, j) = tmp;
+      estGradient(i, j) = (costPlus - costMinus) / (2 * eps);
+    }
+  }
+
+  estGradient = arma::vectorise(estGradient);
+  // Estimate error of gradient.
+  return arma::norm(orgGradient - estGradient) /
+      arma::norm(orgGradient + estGradient);
+}
+
 #endif