sampling layer done, kl divergence forward implemented

src/mlpack/methods/ann/layer/sampling.hpp

@@ -2,7 +2,7 @@
  * @file sampling.hpp
  * @author Atharva Khandait
  *
- * Definition of the Sampling layer class which samples from parameters for a given
+ * Definition of the Sampling layer class which samples from a gaussian
  * distribution.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
@@ -16,13 +16,14 @@
 #include <mlpack/prereqs.hpp>
 
 #include "layer_types.hpp"
+#include "../activation_functions/softplus_function.hpp"
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
 /**
- * Implementation of the Sampling layer class. This layer samples from the given
- * parameters of a normal distribution.
+ * Implementation of the Sampling layer class. This layer samples from the
+ * given parameters of a normal distribution.
  *
  * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
@@ -45,7 +46,7 @@ class Sampling
    * @param inSize The number of input units.
    * @param outSize The number of output units.
    */
-  // Sampling(const size_t inSize, const size_t outSize);
+  Sampling(const size_t inSize, const size_t outSize);
 
   /**
    * Create the Sampling layer object using the specified sample vector size.
@@ -54,11 +55,6 @@ class Sampling
    */
   Sampling(const size_t sampleSize);
 
-  /*
-   * Reset the layer parameter.
-   */
-  // void Reset();
-
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function
    * f(x) by propagating the activity forward through f.
@@ -79,26 +75,33 @@ class Sampling
    * @param g The calculated gradient.
    */
   template<typename eT>
-  void Backward(const arma::Mat<eT>&& /* input */,
+  void Backward(const arma::Mat<eT>&& input,
                 arma::Mat<eT>&& gy,
                 arma::Mat<eT>&& g);
 
-  /*
-   * Calculate the gradient using the output delta and the input activation.
+  /**
+   * Ordinary feed forward pass of a neural network, evaluating
+   * Kullback–Leibler divergence between a normal distribution
+   * and the standard normal.
    *
-   * @param input The input parameter used for calculating the gradient.
-   * @param error The calculated error.
-   * @param gradient The calculated gradient.
+   * @param input Input data used for evaluating the specified function.
+   */
+  template<typename InputType>
+  double klForward();
+
+  /**
+   * Ordinary feed backward pass of a neural network, evaluating the backward
+   * pass of Kullback–Leibler divergence. Using the results from the
+   * KL divergence feed forward pass.
+   *
+   * @param input The propagated input activation.
+   * @param gy The backpropagated error.
+   * @param g The calculated gradient.
    */
   template<typename eT>
-  void Gradient(const arma::Mat<eT>&& input,
-                arma::Mat<eT>&& error,
-                arma::Mat<eT>&& gradient);
-
-  //! Get the parameters.
-  OutputDataType const& Parameters() const { return weights; }
-  //! Modify the parameters.
-  OutputDataType& Parameters() { return weights; }
+  void klBackward(const arma::Mat<eT>&& input,
+                arma::Mat<eT>&& gy,
+                arma::Mat<eT>&& g);
 
   //! Get the input parameter.
   InputDataType const& InputParameter() const { return inputParameter; }
@@ -115,11 +118,6 @@ class Sampling
   //! Modify the delta.
   OutputDataType& Delta() { return delta; }
 
-  //! Get the gradient.
-  OutputDataType const& Gradient() const { return gradient; }
-  //! Modify the gradient.
-  OutputDataType& Gradient() { return gradient; }
-
   //! Get the input size.
   size_t const& InputSize() const { return inSize; }
   //! Modify the input size.
@@ -143,24 +141,18 @@ class Sampling
   //! Locally-stored number of output units.
   size_t outSize;
 
-  //! Locally-stored weight object.
-  OutputDataType weights;
-
-  //! Locally-stored weight parameters.
-  OutputDataType weight;
-
-  //! Locally-stored bias term parameters.
-  OutputDataType bias;
-
   //! Locally-stored delta object.
   OutputDataType delta;
 
-  //! Locally-stored gradient object.
-  OutputDataType gradient;
-
   //! Locally-stored current gaussian sample.
   OutputDataType gaussianSample;
 
+  //! Locally-stored current mean.
+  OutputDataType mean;
+
+  //! Locally-stored current standard deviation.
+  OutputDataType stdDeviation;
+
   //! Locally-stored input parameter object.
   InputDataType inputParameter;
 

src/mlpack/methods/ann/layer/sampling_impl.hpp

@@ -25,15 +25,19 @@ Sampling<InputDataType, OutputDataType>::Sampling()
   // Nothing to do here.
 }
 
-// template <typename InputDataType, typename OutputDataType>
-// Sampling<InputDataType, OutputDataType>::Sampling(
-//     const size_t inSize,
-//     const size_t outSize) :
-//     inSize(inSize),
-//     outSize(outSize)
-// {
-//   weights.set_size(2 * outSize * inSize + 2 * outSize, 1);
-// }
+template <typename InputDataType, typename OutputDataType>
+Sampling<InputDataType, OutputDataType>::Sampling(
+    const size_t inSize,
+    const size_t outSize) :
+    inSize(inSize),
+    outSize(outSize)
+{
+  if (inSize != 2 * outSize)
+  {
+  	Log::Fatal << "The input size of Sampling layer should be 2 * output size!"
+  	    << std::endl;
+  }
+}
 
 template <typename InputDataType, typename OutputDataType>
 Sampling<InputDataType, OutputDataType>::Sampling(
@@ -43,51 +47,45 @@ Sampling<InputDataType, OutputDataType>::Sampling(
   // Nothing to do here.
 }
 
-// template<typename InputDataType, typename OutputDataType>
-// void Sampling<InputDataType, OutputDataType>::Reset()
-// {
-//   weights.set_size(2 * outSize * inSize + 2 * outSize, 1);
-
-//   weight = arma::mat(weights.memptr(), 2 * outSize, inSize, false, false);
-//   bias = arma::mat(weights.memptr() + weight.n_elem,
-//       2 * outSize, 1, false, false);
-// }
-
 template<typename InputDataType, typename OutputDataType>
 template<typename eT>
 void Sampling<InputDataType, OutputDataType>::Forward(
     const arma::Mat<eT>&& input, arma::Mat<eT>&& output)
 {
+  if (input.n_rows != 2 * outSize)
+  {
+  	Log::Fatal << "The output size of layer before the Sampling layer should "
+  	    << "be 2 * output size of the Sampling layer!" << std::endl;
+  }
+
   arma::arma_rng::set_seed_random();
 
-  // output = weight * input;
-  // output.each_col() += bias;
-  gaussianSample = arma::randn<arma::mat>(outSize, input.n_cols);
-  output = (input.submat(outSize, 0, 2 * outSize - 1, input.n_cols - 1) +
-      input.submat(0, 0, outSize - 1, input.n_cols - 1)) % gaussianSample;
+  mean = input.submat(outSize, 0, 2 * outSize - 1, input.n_cols - 1);
+  SoftplusFunction::Fn(input.submat(0, 0, outSize - 1, input.n_cols - 1),
+      stdDeviation);
+
+  gaussianSample = arma::randn<arma::Mat<eT>>(outSize, input.n_cols);
+  output = mean + stdDeviation % gaussianSample;
 }
 
 template<typename InputDataType, typename OutputDataType>
 template<typename eT>
 void Sampling<InputDataType, OutputDataType>::Backward(
-    const arma::Mat<eT>&& /* input */, arma::Mat<eT>&& gy, arma::Mat<eT>&& g)
-{std::cout << gy.n_rows << std::endl << gy.n_cols << std::endl;
-  g = join_cols((weight.submat(0, 0, outSize - 1, inSize - 1).t() * gy) % 
-      gaussianSample, 
-      weight.submat(outSize, 0, 2 * outSize - 1, inSize - 1).t() * gy);
+    const arma::Mat<eT>&& input, arma::Mat<eT>&& gy, arma::Mat<eT>&& g)
+{ 
+  arma::Mat<eT> softplusDer;
+  SoftplusFunction::Deriv((input - mean) / gaussianSample, 
+      softplusDer);
+
+  g = join_cols(gy % std::move(gaussianSample) % std::move(softplusDer), gy);
 }
 
 template<typename InputDataType, typename OutputDataType>
-template<typename eT>
-void Sampling<InputDataType, OutputDataType>::Gradient(
-    const arma::Mat<eT>&& input,
-    arma::Mat<eT>&& error,
-    arma::Mat<eT>&& gradient)
+template<typename InputType>
+double Sampling<InputDataType, OutputDataType>::klForward()
 {
-  gradient.submat(0, 0, weight.n_elem - 1, 0) = arma::vectorise(
-      error * input.t());
-  gradient.submat(weight.n_elem, 0, gradient.n_elem - 1, 0) =
-      arma::sum(error, 1);
+  return -0.5 * arma::accu(arma::log(stdDeviation) - stdDeviation -
+      arma::pow(mean, 2) + 1);
 }
 
 template<typename InputDataType, typename OutputDataType>
@@ -95,13 +93,8 @@ template<typename Archive>
 void Sampling<InputDataType, OutputDataType>::serialize(
     Archive& ar, const unsigned int /* version */)
 {
-  ar & BOOST_SERIALIZATION_NVP(inSize);
+  // ar & BOOST_SERIALIZATION_NVP(inSize);
   ar & BOOST_SERIALIZATION_NVP(outSize);
-
-  // This is inefficient, but we have to allocate this memory so that
-  // WeightSetVisitor gets the right size.
-  if (Archive::is_loading::value)
-    weights.set_size(2 * outSize * inSize + 2 * outSize, 1);
 }
 
 } // namespace ann

src/mlpack/tests/ann_layer_test.cpp

@@ -1847,4 +1847,21 @@ BOOST_AUTO_TEST_CASE(SubviewIndexTest)
   CheckMatrices(outputEnd, subEnd);
 }
 
+/*
+ * Simple Sampling module test.
+ */
+BOOST_AUTO_TEST_CASE(SimpleSamplingLayerTest)
+{
+  arma::mat input, output, delta;
+  Sampling<> module(10, 5);
+
+  // Test the Forward function.
+  input = join_cols(arma::ones<arma::mat>(5, 1) * -10,
+      arma::zeros<arma::mat>(5, 1));
+  module.Forward(std::move(input), std::move(output));
+  BOOST_REQUIRE_LE(arma::accu(output), 1e-3);
+
+  // Test the Backward function.
+}
+
 BOOST_AUTO_TEST_SUITE_END();