Add reconstruction loss with test.

src/mlpack/methods/ann/loss_functions/CMakeLists.txt

@@ -11,6 +11,8 @@ set(SOURCES
   mean_squared_error_impl.hpp
   negative_log_likelihood.hpp
   negative_log_likelihood_impl.hpp
+  reconstruction_loss.hpp
+  reconstruction_loss_impl.hpp
   sigmoid_cross_entropy_error.hpp
   sigmoid_cross_entropy_error_impl.hpp
 )

src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp

@@ -0,0 +1,89 @@
+/**
+ * @file reconstruction_loss.hpp
+ * @author Atharva Khandait
+ *
+ * Definition of the reconstruction loss performance function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_RECONSTRUCTION_LOSS_HPP
+#define MLPACK_METHODS_ANN_LOSS_FUNCTION_RECONSTRUCTION_LOSS_HPP
+
+#include <mlpack/prereqs.hpp>
+#include <mlpack/methods/ann/dists/normal_distribution.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * The mean squared error performance function measures the network's
+ * performance according to the mean of squared errors.
+ *
+ * @tparam ActivationFunction Activation function used for the embedding layer.
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ */
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat,
+    typename DistType = NormalDistribution<InputDataType>
+>
+class ReconstructionLoss
+{
+ public:
+  /**
+   * Create the ReconstructionLoss object.
+   */
+  ReconstructionLoss();
+
+  /*
+   * Computes the mean squared error function.
+   *
+   * @param input Input data used for evaluating the specified function.
+   * @param output Resulting output activation.
+   */
+  template<typename InputType, typename TargetType>
+  double Forward(const InputType&& input, const TargetType&& target);
+  /**
+   * Ordinary feed backward pass of a neural network.
+   *
+   * @param input The propagated input activation.
+   * @param target The target vector.
+   * @param output The calculated error.
+   */
+  template<typename InputType, typename TargetType, typename OutputType>
+  void Backward(const InputType&& input,
+                const TargetType&& target,
+                OutputType&& output);
+
+  //! Get the output parameter.
+  OutputDataType& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  /**
+   * Serialize the layer
+   */
+  template<typename Archive>
+  void serialize(Archive& ar, const unsigned int /* version */);
+
+ private:
+  //! Locally-stored distribution object.
+  DistType* dist;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+}; // class ReconstructionLoss
+
+} // namespace ann
+} // namespace mlpack
+
+// Include implementation.
+#include "reconstruction_loss_impl.hpp"
+
+#endif

src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp

@@ -0,0 +1,62 @@
+/**
+ * @file reconstruction_loss_impl.hpp
+ * @author Atharva Khandait
+ *
+ * Implementation of the reconstruction loss performance function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_RECONSTRUCTION_LOSS_IMPL_HPP
+#define MLPACK_METHODS_ANN_LOSS_FUNCTION_RECONSTRUCTION_LOSS_IMPL_HPP
+
+// In case it hasn't yet been included.
+#include "reconstruction_loss.hpp"
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+template<typename InputDataType, typename OutputDataType, typename DistType>
+ReconstructionLoss<
+    InputDataType,
+    OutputDataType,
+    DistType
+>::ReconstructionLoss()
+{
+  // Nothing to do here.
+}
+
+template<typename InputDataType, typename OutputDataType, typename DistType>
+template<typename InputType, typename TargetType>
+double ReconstructionLoss<InputDataType, OutputDataType, DistType>::Forward(
+    const InputType&& input, const TargetType&& target)
+{
+  dist = new DistType(std::move(input));
+  return dist->LogProbability(std::move(target));
+}
+
+template<typename InputDataType, typename OutputDataType, typename DistType>
+template<typename InputType, typename TargetType, typename OutputType>
+void ReconstructionLoss<InputDataType, OutputDataType, DistType>::Backward(
+    const InputType&& /* input */,
+    const TargetType&& target,
+    OutputType&& output)
+{
+  dist->LogProbBackward(std::move(target), std::move(output));
+}
+
+template<typename InputDataType, typename OutputDataType, typename DistType>
+template<typename Archive>
+void ReconstructionLoss<InputDataType, OutputDataType, DistType>::serialize(
+    Archive& /* ar */,
+    const unsigned int /* version */)
+{
+  // Nothing to do here.
+}
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/tests/CMakeLists.txt

@@ -9,6 +9,7 @@ add_executable(mlpack_test
   aknn_test.cpp
   ann_dist_test.cpp
   ann_layer_test.cpp
+  ann_test_tools.hpp
   arma_extend_test.cpp
   armadillo_svd_test.cpp
   async_learning_test.cpp

src/mlpack/tests/ann_layer_test.cpp

@@ -14,8 +14,6 @@
 
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/ann/layer/layer_types.hpp>
-#include <mlpack/methods/ann/loss_functions/mean_squared_error.hpp>
-#include <mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp>
 #include <mlpack/methods/ann/init_rules/random_init.hpp>
 #include <mlpack/methods/ann/init_rules/const_init.hpp>
 #include <mlpack/methods/ann/init_rules/nguyen_widrow_init.hpp>
@@ -24,162 +22,13 @@
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
+#include "ann_test_tools.hpp"
 
 using namespace mlpack;
 using namespace mlpack::ann;
 
 BOOST_AUTO_TEST_SUITE(ANNLayerTest);
 
-// Helper function which calls the Reset function of the given module.
-template<class T>
-void ResetFunction(
-    T& layer,
-    typename std::enable_if<HasResetCheck<T, void(T::*)()>::value>::type* = 0)
-{
-  layer.Reset();
-}
-
-template<class T>
-void ResetFunction(
-    T& /* layer */,
-    typename std::enable_if<!HasResetCheck<T, void(T::*)()>::value>::type* = 0)
-{
-  /* Nothing to do here */
-}
-
-// Approximate Jacobian and supposedly-true Jacobian, then compare them
-// similarly to before.
-template<typename ModuleType>
-double JacobianTest(ModuleType& module,
-                  arma::mat& input,
-                  const double minValue = -2,
-                  const double maxValue = -1,
-                  const double perturbation = 1e-6)
-{
-  arma::mat output, outputA, outputB, jacobianA, jacobianB;
-
-  // Initialize the input matrix.
-  RandomInitialization init(minValue, maxValue);
-  init.Initialize(input, input.n_rows, input.n_cols);
-
-  // Initialize the module parameters.
-  ResetFunction(module);
-
-  // Initialize the jacobian matrix.
-  module.Forward(std::move(input), std::move(output));
-  jacobianA = arma::zeros(input.n_elem, output.n_elem);
-
-  // Share the input paramter matrix.
-  arma::mat sin = arma::mat(input.memptr(), input.n_rows, input.n_cols,
-      false, false);
-
-  for (size_t i = 0; i < input.n_elem; ++i)
-  {
-    double original = sin(i);
-    sin(i) = original - perturbation;
-    module.Forward(std::move(input), std::move(outputA));
-    sin(i) = original + perturbation;
-    module.Forward(std::move(input), std::move(outputB));
-    sin(i) = original;
-
-    outputB -= outputA;
-    outputB /= 2 * perturbation;
-    jacobianA.row(i) = outputB.t();
-  }
-
-  // Initialize the derivative parameter.
-  arma::mat deriv = arma::zeros(output.n_rows, output.n_cols);
-
-  // Share the derivative parameter.
-  arma::mat derivTemp = arma::mat(deriv.memptr(), deriv.n_rows, deriv.n_cols,
-      false, false);
-
-  // Initialize the jacobian matrix.
-  jacobianB = arma::zeros(input.n_elem, output.n_elem);
-
-  for (size_t i = 0; i < derivTemp.n_elem; ++i)
-  {
-    deriv.zeros();
-    derivTemp(i) = 1;
-
-    arma::mat delta;
-    module.Backward(std::move(input), std::move(deriv), std::move(delta));
-
-    jacobianB.col(i) = delta;
-  }
-
-  return arma::max(arma::max(arma::abs(jacobianA - jacobianB)));
-}
-
-// Approximate Jacobian and supposedly-true Jacobian, then compare them
-// similarly to before.
-template<typename ModuleType>
-double JacobianPerformanceTest(ModuleType& module,
-                               arma::mat& input,
-                               arma::mat& target,
-                               const double eps = 1e-6)
-{
-  module.Forward(std::move(input), std::move(target));
-
-  arma::mat delta;
-  module.Backward(std::move(input), std::move(target), std::move(delta));
-
-  arma::mat centralDifference = arma::zeros(delta.n_rows, delta.n_cols);
-  arma::mat inputTemp = arma::mat(input.memptr(), input.n_rows, input.n_cols,
-      false, false);
-
-  arma::mat centralDifferenceTemp = arma::mat(centralDifference.memptr(),
-      centralDifference.n_rows, centralDifference.n_cols, false, false);
-
-  for (size_t i = 0; i < input.n_elem; ++i)
-  {
-    inputTemp(i) = inputTemp(i) + eps;
-    double outputA = module.Forward(std::move(input), std::move(target));
-    inputTemp(i) = inputTemp(i) - (2 * eps);
-    double outputB = module.Forward(std::move(input), std::move(target));
-
-    centralDifferenceTemp(i) = (outputA - outputB) / (2 * eps);
-    inputTemp(i) = inputTemp(i) + eps;
-  }
-
-  return arma::max(arma::max(arma::abs(centralDifference - delta)));
-}
-
-// Simple numerical gradient checker.
-template<class FunctionType>
-double CheckGradient(FunctionType& function, const double eps = 1e-7)
-{
-  // Get gradients for the current parameters.
-  arma::mat orgGradient, gradient, estGradient;
-  function.Gradient(orgGradient);
-
-  estGradient = arma::zeros(orgGradient.n_rows, orgGradient.n_cols);
-
-  // Compute numeric approximations to gradient.
-  for (size_t i = 0; i < orgGradient.n_elem; ++i)
-  {
-    double tmp = function.Parameters()(i);
-
-    // Perturb parameter with a positive constant and get costs.
-    function.Parameters()(i) += eps;
-    double costPlus = function.Gradient(gradient);
-
-    // Perturb parameter with a negative constant and get costs.
-    function.Parameters()(i) -= (2 * eps);
-    double costMinus = function.Gradient(gradient);
-
-    // Restore the parameter value.
-    function.Parameters()(i) = tmp;
-
-    // Compute numerical gradients using the costs calculated above.
-    estGradient(i) = (costPlus - costMinus) / (2 * eps);
-  }
-
-  // Estimate error of gradient.
-  return arma::norm(orgGradient - estGradient) /
-      arma::norm(orgGradient + estGradient);
-}
-
 /**
  * Simple add module test.
  */

src/mlpack/tests/ann_test_tools.hpp

@@ -0,0 +1,170 @@
+/**
+ * @file ann_test_tools.hpp
+ * @author Marcus Edel
+ *
+ * This file includes some useful functions for ann tests.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_TESTS_ANN_TEST_TOOLS_HPP
+#define MLPACK_TESTS_ANN_TEST_TOOLS_HPP
+
+#include <mlpack/core.hpp>
+
+using namespace mlpack;
+using namespace mlpack::ann;
+
+// Helper function which calls the Reset function of the given module.
+template<class T>
+void ResetFunction(
+    T& layer,
+    typename std::enable_if<HasResetCheck<T, void(T::*)()>::value>::type* = 0)
+{
+  layer.Reset();
+}
+
+template<class T>
+void ResetFunction(
+    T& /* layer */,
+    typename std::enable_if<!HasResetCheck<T, void(T::*)()>::value>::type* = 0)
+{
+  /* Nothing to do here */
+}
+
+// Approximate Jacobian and supposedly-true Jacobian, then compare them
+// similarly to before.
+template<typename ModuleType>
+double JacobianTest(ModuleType& module,
+                  arma::mat& input,
+                  const double minValue = -2,
+                  const double maxValue = -1,
+                  const double perturbation = 1e-6)
+{
+  arma::mat output, outputA, outputB, jacobianA, jacobianB;
+
+  // Initialize the input matrix.
+  RandomInitialization init(minValue, maxValue);
+  init.Initialize(input, input.n_rows, input.n_cols);
+
+  // Initialize the module parameters.
+  ResetFunction(module);
+
+  // Initialize the jacobian matrix.
+  module.Forward(std::move(input), std::move(output));
+  jacobianA = arma::zeros(input.n_elem, output.n_elem);
+
+  // Share the input paramter matrix.
+  arma::mat sin = arma::mat(input.memptr(), input.n_rows, input.n_cols,
+      false, false);
+
+  for (size_t i = 0; i < input.n_elem; ++i)
+  {
+    double original = sin(i);
+    sin(i) = original - perturbation;
+    module.Forward(std::move(input), std::move(outputA));
+    sin(i) = original + perturbation;
+    module.Forward(std::move(input), std::move(outputB));
+    sin(i) = original;
+
+    outputB -= outputA;
+    outputB /= 2 * perturbation;
+    jacobianA.row(i) = outputB.t();
+  }
+
+  // Initialize the derivative parameter.
+  arma::mat deriv = arma::zeros(output.n_rows, output.n_cols);
+
+  // Share the derivative parameter.
+  arma::mat derivTemp = arma::mat(deriv.memptr(), deriv.n_rows, deriv.n_cols,
+      false, false);
+
+  // Initialize the jacobian matrix.
+  jacobianB = arma::zeros(input.n_elem, output.n_elem);
+
+  for (size_t i = 0; i < derivTemp.n_elem; ++i)
+  {
+    deriv.zeros();
+    derivTemp(i) = 1;
+
+    arma::mat delta;
+    module.Backward(std::move(input), std::move(deriv), std::move(delta));
+
+    jacobianB.col(i) = delta;
+  }
+
+  return arma::max(arma::max(arma::abs(jacobianA - jacobianB)));
+}
+
+// Approximate Jacobian and supposedly-true Jacobian, then compare them
+// similarly to before.
+template<typename ModuleType>
+double JacobianPerformanceTest(ModuleType& module,
+                               arma::mat& input,
+                               arma::mat& target,
+                               const double eps = 1e-6)
+{
+  module.Forward(std::move(input), std::move(target));
+
+  arma::mat delta;
+  module.Backward(std::move(input), std::move(target), std::move(delta));
+
+  arma::mat centralDifference = arma::zeros(delta.n_rows, delta.n_cols);
+  arma::mat inputTemp = arma::mat(input.memptr(), input.n_rows, input.n_cols,
+      false, false);
+
+  arma::mat centralDifferenceTemp = arma::mat(centralDifference.memptr(),
+      centralDifference.n_rows, centralDifference.n_cols, false, false);
+
+  for (size_t i = 0; i < input.n_elem; ++i)
+  {
+    inputTemp(i) = inputTemp(i) + eps;
+    double outputA = module.Forward(std::move(input), std::move(target));
+    inputTemp(i) = inputTemp(i) - (2 * eps);
+    double outputB = module.Forward(std::move(input), std::move(target));
+
+    centralDifferenceTemp(i) = (outputA - outputB) / (2 * eps);
+    inputTemp(i) = inputTemp(i) + eps;
+  }
+
+  return arma::max(arma::max(arma::abs(centralDifference - delta)));
+}
+
+// Simple numerical gradient checker.
+template<class FunctionType>
+double CheckGradient(FunctionType& function, const double eps = 1e-7)
+{
+  // Get gradients for the current parameters.
+  arma::mat orgGradient, gradient, estGradient;
+  function.Gradient(orgGradient);
+
+  estGradient = arma::zeros(orgGradient.n_rows, orgGradient.n_cols);
+
+  // Compute numeric approximations to gradient.
+  for (size_t i = 0; i < orgGradient.n_elem; ++i)
+  {
+    double tmp = function.Parameters()(i);
+
+    // Perturb parameter with a positive constant and get costs.
+    function.Parameters()(i) += eps;
+    double costPlus = function.Gradient(gradient);
+
+    // Perturb parameter with a negative constant and get costs.
+    function.Parameters()(i) -= (2 * eps);
+    double costMinus = function.Gradient(gradient);
+
+    // Restore the parameter value.
+    function.Parameters()(i) = tmp;
+
+    // Compute numerical gradients using the costs calculated above.
+    estGradient(i) = (costPlus - costMinus) / (2 * eps);
+  }
+
+  // Estimate error of gradient.
+  return arma::norm(orgGradient - estGradient) /
+      arma::norm(orgGradient + estGradient);
+}
+
+#endif

src/mlpack/tests/loss_functions_test.cpp

@@ -2,6 +2,7 @@
  * @file loss_functions_test.cpp
  * @author Dakshit Agrawal
  * @author Sourabh Varshney
+ * @author Atharva Khandait
  *
  * Tests for loss functions in mlpack::methods::ann:loss_functions.
  *
@@ -10,14 +11,21 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#include <mlpack/methods/ann/loss_functions/earth_mover_distance.hpp>
+#include <mlpack/core.hpp>
+
+#include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/ann/loss_functions/kl_divergence.hpp>
+#include <mlpack/methods/ann/loss_functions/earth_mover_distance.hpp>
 #include <mlpack/methods/ann/loss_functions/mean_squared_error.hpp>
 #include <mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp>
 #include <mlpack/methods/ann/loss_functions/cross_entropy_error.hpp>
+#include <mlpack/methods/ann/loss_functions/reconstruction_loss.hpp>
+#include <mlpack/methods/ann/init_rules/nguyen_widrow_init.hpp>
+#include <mlpack/methods/ann/ffn.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
+#include "ann_test_tools.hpp"
 
 using namespace mlpack;
 using namespace mlpack::ann;
@@ -85,7 +93,7 @@ BOOST_AUTO_TEST_CASE(KLDivergenceNoMeanTest)
 /*
  * Simple test for the mean squared error performance function.
  */
-BOOST_AUTO_TEST_CASE(SimpleMeanSquaredErrorLayerTest)
+BOOST_AUTO_TEST_CASE(SimpleMeanSquaredErrorTest)
 {
   arma::mat input, output, target;
   MeanSquaredError<> module;
@@ -120,7 +128,7 @@ BOOST_AUTO_TEST_CASE(SimpleMeanSquaredErrorLayerTest)
 /*
  * Simple test for the cross-entropy error performance function.
  */
-BOOST_AUTO_TEST_CASE(SimpleCrossEntropyErrorLayerTest)
+BOOST_AUTO_TEST_CASE(SimpleCrossEntropyErrorTest)
 {
   arma::mat input1, input2, output, target1, target2;
   CrossEntropyError<> module(1e-6);
@@ -161,9 +169,9 @@ BOOST_AUTO_TEST_CASE(SimpleCrossEntropyErrorLayerTest)
 }
 
 /**
- * Simple test for the Sigmoid Cross Entropy Layer.
+ * Simple test for the Sigmoid Cross Entropy performance function.
  */
-BOOST_AUTO_TEST_CASE(SimpleSigmoidCrossEntropyLayerTest)
+BOOST_AUTO_TEST_CASE(SimpleSigmoidCrossEntropyErrorTest)
 {
   arma::mat input1, input2, input3, output, target1,
             target2, target3, expectedOutput;
@@ -221,6 +229,7 @@ BOOST_AUTO_TEST_CASE(SimpleSigmoidCrossEntropyLayerTest)
 }
 
 /**
+<<<<<<< 9cecc3e18160a4411c1e80831b6773bdbd80b789
  * Simple test for the Earth Mover Distance Layer.
  */
 BOOST_AUTO_TEST_CASE(SimpleEarthMoverDistanceLayerTest)
@@ -256,6 +265,48 @@ BOOST_AUTO_TEST_CASE(SimpleEarthMoverDistanceLayerTest)
     BOOST_REQUIRE_SMALL(output(i) - expectedOutput(i), 1e-5);
   BOOST_REQUIRE_EQUAL(output.n_rows, input2.n_rows);
   BOOST_REQUIRE_EQUAL(output.n_cols, input2.n_cols);
+=======
+ * Reconstruction Loss numerical gradient test.
+ */
+BOOST_AUTO_TEST_CASE(GradientReconstructionLossTest)
+{
+  // Linear function gradient instantiation.
+  struct GradientFunction
+  {
+    GradientFunction()
+    {
+      input = arma::randu(10, 1);
+      target = arma::randu(2, 1);
+
+      model = new FFN<ReconstructionLoss<>, NguyenWidrowInitialization>();
+      model->Predictors() = input;
+      model->Responses() = target;
+      model->Add<IdentityLayer<> >();
+      model->Add<Linear<> >(10, 4);
+      model->Add<ReLULayer<> >();
+    }
+
+    ~GradientFunction()
+    {
+      delete model;
+    }
+
+    double Gradient(arma::mat& gradient) const
+    {
+      arma::mat output;
+      double error = model->Evaluate(model->Parameters(), 0, 1);
+      model->Gradient(model->Parameters(), 0, gradient, 1);
+      return error;
+    }
+
+    arma::mat& Parameters() { return model->Parameters(); }
+
+    FFN<ReconstructionLoss<>, NguyenWidrowInitialization>* model;
+    arma::mat input, target;
+  } function;
+
+  BOOST_REQUIRE_LE(CheckGradient(function), 1e-4);
+>>>>>>> Add reconstruction loss with test.
 }
 
 BOOST_AUTO_TEST_SUITE_END();