From 3a7bb6a3f69d4bd180df6c9bc37c567341dfd566 Mon Sep 17 00:00:00 2001 From: akhandait Date: Mon, 4 Jun 2018 12:54:32 +0530 Subject: [PATCH] sampling layer done, kl divergence forward implemented --- src/mlpack/methods/ann/layer/sampling.hpp | 72 ++++++++--------- .../methods/ann/layer/sampling_impl.hpp | 81 +++++++++---------- src/mlpack/tests/ann_layer_test.cpp | 17 ++++ 3 files changed, 86 insertions(+), 84 deletions(-) diff --git a/src/mlpack/methods/ann/layer/sampling.hpp b/src/mlpack/methods/ann/layer/sampling.hpp index 9beec42f6e..61df2c7da7 100644 --- a/src/mlpack/methods/ann/layer/sampling.hpp +++ b/src/mlpack/methods/ann/layer/sampling.hpp @@ -2,7 +2,7 @@ * @file sampling.hpp * @author Atharva Khandait * - * Definition of the Sampling layer class which samples from parameters for a given + * Definition of the Sampling layer class which samples from a gaussian * distribution. * * mlpack is free software; you may redistribute it and/or modify it under the @@ -16,13 +16,14 @@ #include #include "layer_types.hpp" +#include "../activation_functions/softplus_function.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * Implementation of the Sampling layer class. This layer samples from the given - * parameters of a normal distribution. + * Implementation of the Sampling layer class. This layer samples from the + * given parameters of a normal distribution. * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). @@ -45,7 +46,7 @@ class Sampling * @param inSize The number of input units. * @param outSize The number of output units. */ - // Sampling(const size_t inSize, const size_t outSize); + Sampling(const size_t inSize, const size_t outSize); /** * Create the Sampling layer object using the specified sample vector size. @@ -54,11 +55,6 @@ class Sampling */ Sampling(const size_t sampleSize); - /* - * Reset the layer parameter. - */ - // void Reset(); - /** * Ordinary feed forward pass of a neural network, evaluating the function * f(x) by propagating the activity forward through f. @@ -79,26 +75,33 @@ class Sampling * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, + void Backward(const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g); - /* - * Calculate the gradient using the output delta and the input activation. + /** + * Ordinary feed forward pass of a neural network, evaluating + * Kullback–Leibler divergence between a normal distribution + * and the standard normal. * - * @param input The input parameter used for calculating the gradient. - * @param error The calculated error. - * @param gradient The calculated gradient. + * @param input Input data used for evaluating the specified function. + */ + template + double klForward(); + + /** + * Ordinary feed backward pass of a neural network, evaluating the backward + * pass of Kullback–Leibler divergence. Using the results from the + * KL divergence feed forward pass. + * + * @param input The propagated input activation. + * @param gy The backpropagated error. + * @param g The calculated gradient. */ template - void Gradient(const arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient); - - //! Get the parameters. - OutputDataType const& Parameters() const { return weights; } - //! Modify the parameters. - OutputDataType& Parameters() { return weights; } + void klBackward(const arma::Mat&& input, + arma::Mat&& gy, + arma::Mat&& g); //! Get the input parameter. InputDataType const& InputParameter() const { return inputParameter; } @@ -115,11 +118,6 @@ class Sampling //! Modify the delta. OutputDataType& Delta() { return delta; } - //! Get the gradient. - OutputDataType const& Gradient() const { return gradient; } - //! Modify the gradient. - OutputDataType& Gradient() { return gradient; } - //! Get the input size. size_t const& InputSize() const { return inSize; } //! Modify the input size. @@ -143,24 +141,18 @@ class Sampling //! Locally-stored number of output units. size_t outSize; - //! Locally-stored weight object. - OutputDataType weights; - - //! Locally-stored weight parameters. - OutputDataType weight; - - //! Locally-stored bias term parameters. - OutputDataType bias; - //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored gradient object. - OutputDataType gradient; - //! Locally-stored current gaussian sample. OutputDataType gaussianSample; + //! Locally-stored current mean. + OutputDataType mean; + + //! Locally-stored current standard deviation. + OutputDataType stdDeviation; + //! Locally-stored input parameter object. InputDataType inputParameter; diff --git a/src/mlpack/methods/ann/layer/sampling_impl.hpp b/src/mlpack/methods/ann/layer/sampling_impl.hpp index 48be2a8c4f..3d1b11dec0 100644 --- a/src/mlpack/methods/ann/layer/sampling_impl.hpp +++ b/src/mlpack/methods/ann/layer/sampling_impl.hpp @@ -25,15 +25,19 @@ Sampling::Sampling() // Nothing to do here. } -// template -// Sampling::Sampling( -// const size_t inSize, -// const size_t outSize) : -// inSize(inSize), -// outSize(outSize) -// { -// weights.set_size(2 * outSize * inSize + 2 * outSize, 1); -// } +template +Sampling::Sampling( + const size_t inSize, + const size_t outSize) : + inSize(inSize), + outSize(outSize) +{ + if (inSize != 2 * outSize) + { + Log::Fatal << "The input size of Sampling layer should be 2 * output size!" + << std::endl; + } +} template Sampling::Sampling( @@ -43,51 +47,45 @@ Sampling::Sampling( // Nothing to do here. } -// template -// void Sampling::Reset() -// { -// weights.set_size(2 * outSize * inSize + 2 * outSize, 1); - -// weight = arma::mat(weights.memptr(), 2 * outSize, inSize, false, false); -// bias = arma::mat(weights.memptr() + weight.n_elem, -// 2 * outSize, 1, false, false); -// } - template template void Sampling::Forward( const arma::Mat&& input, arma::Mat&& output) { + if (input.n_rows != 2 * outSize) + { + Log::Fatal << "The output size of layer before the Sampling layer should " + << "be 2 * output size of the Sampling layer!" << std::endl; + } + arma::arma_rng::set_seed_random(); - // output = weight * input; - // output.each_col() += bias; - gaussianSample = arma::randn(outSize, input.n_cols); - output = (input.submat(outSize, 0, 2 * outSize - 1, input.n_cols - 1) + - input.submat(0, 0, outSize - 1, input.n_cols - 1)) % gaussianSample; + mean = input.submat(outSize, 0, 2 * outSize - 1, input.n_cols - 1); + SoftplusFunction::Fn(input.submat(0, 0, outSize - 1, input.n_cols - 1), + stdDeviation); + + gaussianSample = arma::randn>(outSize, input.n_cols); + output = mean + stdDeviation % gaussianSample; } template template void Sampling::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) -{std::cout << gy.n_rows << std::endl << gy.n_cols << std::endl; - g = join_cols((weight.submat(0, 0, outSize - 1, inSize - 1).t() * gy) % - gaussianSample, - weight.submat(outSize, 0, 2 * outSize - 1, inSize - 1).t() * gy); + const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) +{ + arma::Mat softplusDer; + SoftplusFunction::Deriv((input - mean) / gaussianSample, + softplusDer); + + g = join_cols(gy % std::move(gaussianSample) % std::move(softplusDer), gy); } template -template -void Sampling::Gradient( - const arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient) +template +double Sampling::klForward() { - gradient.submat(0, 0, weight.n_elem - 1, 0) = arma::vectorise( - error * input.t()); - gradient.submat(weight.n_elem, 0, gradient.n_elem - 1, 0) = - arma::sum(error, 1); + return -0.5 * arma::accu(arma::log(stdDeviation) - stdDeviation - + arma::pow(mean, 2) + 1); } template @@ -95,13 +93,8 @@ template void Sampling::serialize( Archive& ar, const unsigned int /* version */) { - ar & BOOST_SERIALIZATION_NVP(inSize); + // ar & BOOST_SERIALIZATION_NVP(inSize); ar & BOOST_SERIALIZATION_NVP(outSize); - - // This is inefficient, but we have to allocate this memory so that - // WeightSetVisitor gets the right size. - if (Archive::is_loading::value) - weights.set_size(2 * outSize * inSize + 2 * outSize, 1); } } // namespace ann diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index aaae439aec..04d26600cc 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -1847,4 +1847,21 @@ BOOST_AUTO_TEST_CASE(SubviewIndexTest) CheckMatrices(outputEnd, subEnd); } +/* + * Simple Sampling module test. + */ +BOOST_AUTO_TEST_CASE(SimpleSamplingLayerTest) +{ + arma::mat input, output, delta; + Sampling<> module(10, 5); + + // Test the Forward function. + input = join_cols(arma::ones(5, 1) * -10, + arma::zeros(5, 1)); + module.Forward(std::move(input), std::move(output)); + BOOST_REQUIRE_LE(arma::accu(output), 1e-3); + + // Test the Backward function. +} + BOOST_AUTO_TEST_SUITE_END();