From dfeda93e4ba9cdc266cd8c6d8446f58c003cb28b Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Wed, 21 Mar 2018 12:57:42 +0100 Subject: [PATCH] Update backward step (pass the gradient check). --- src/mlpack/methods/ann/layer/batch_norm.hpp | 18 ++++--- .../methods/ann/layer/batch_norm_impl.hpp | 52 +++++++++---------- src/mlpack/tests/ann_layer_test.cpp | 28 +--------- 3 files changed, 38 insertions(+), 60 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batch_norm.hpp b/src/mlpack/methods/ann/layer/batch_norm.hpp index 1bd4e1afb0..469dac1bd7 100644 --- a/src/mlpack/methods/ann/layer/batch_norm.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm.hpp @@ -3,7 +3,7 @@ * @author Praveen Ch * @author Manthan-R-Sheth * - * Definition of the Batch Normalisation layer class + * Definition of the Batch Normalization layer class. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -20,7 +20,7 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * Declaration of the Batch Normalisation layer class. The layer tranforms + * Declaration of the Batch Normalization layer class. The layer tranforms * the input data into zero mean and unit variance and then scales and shifts * the data by parameters, gamma and beta respectively. These parameters are * learnt by the network. @@ -32,15 +32,17 @@ namespace ann /** Artificial Neural Network. */ { * For more information, refer to the following paper, * * @code - * @article{DBLP:journals/corr/IoffeS15, + * @article{Ioffe15, * author = {Sergey Ioffe and * Christian Szegedy}, * title = {Batch Normalization: Accelerating Deep Network Training by * Reducing Internal Covariate Shift}, * journal = {CoRR}, - * volume = {abs/1502.03167} + * volume = {abs/1502.03167}, + * year = {2015}, + * url = {http://arxiv.org/abs/1502.03167}, + * eprint = {1502.03167}, * } - * * @endcode * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, @@ -48,7 +50,6 @@ namespace ann /** Artificial Neural Network. */ { * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ - template < typename InputDataType = arma::mat, typename OutputDataType = arma::mat @@ -65,7 +66,7 @@ class BatchNorm * @param size The number of input units. * @param eps The epsilon added to variance to ensure numerical stability. */ - BatchNorm(const size_t size, const double eps = 0.001); + BatchNorm(const size_t size, const double eps = 1e-8); /** * Reset the layer parameters @@ -191,6 +192,9 @@ class BatchNorm //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Locally-stored normalized input. + OutputDataType normalized; }; // class BatchNorm } // namespace ann diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index 9d5dd4bfd6..8bceffe3fc 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -23,7 +23,7 @@ namespace ann { /** Artificial Neural Network. */ template BatchNorm::BatchNorm() : size(10), - eps(1e-7), + eps(1e-8), deterministic(false) { // Nothing to do here. @@ -55,29 +55,34 @@ template void BatchNorm::Forward( const arma::Mat&& input, arma::Mat&& output) { - output.reshape(input.n_rows, input.n_cols); - // Mean and variance over the entire training set will be used to compute // the forward pass when deterministic is set to true. if (deterministic) { mean = stats.mean(); variance = stats.var(1); + + output = input.each_col() - mean; + output.each_col() %= gamma / arma::sqrt(variance + eps); + output.each_col() += beta; } else { mean = arma::mean(input, 1); variance = arma::var(input, 1, 1); - for (size_t i = 0; i < output.n_cols; i++) - { + for (size_t i = 0; i < input.n_cols; i++) stats(input.col(i)); - } - } - output = input.each_col() - mean; - output.each_col() %= gamma / arma::sqrt(variance + eps); - output.each_col() += beta; + output = input.each_col() - mean; + output.each_col() /= arma::sqrt(variance + eps); + + // Reused in the backward and gradient step. + normalized = output; + + output.each_col() %= gamma; + output.each_col() += beta; + } } template @@ -85,34 +90,29 @@ template void BatchNorm::Backward( const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) { - mean = arma::mean(input, 1); - variance = arma::var(input, 1, 1); + const arma::mat inputMean = input.each_col() - mean; + const arma::mat stdInv = 1.0 / arma::sqrt(variance + eps); + const arma::mat norm = gy.each_col() % gamma; + const arma::mat var = arma::sum(norm % inputMean, 1) % + arma::pow(stdInv, 3.0) * 0.5; - arma::mat m = arma::sum(gy % (input.each_col() - mean), 1); - g = (mean - input.each_col()); - g.each_col() %= m; - g.each_col() %= 1.0/(variance + eps); - g += (gy.each_col() - arma::sum(gy, 1)); - g += (input.n_cols - 1) * gy; - g.each_col() %= ((1.0 / input.n_cols) * gamma); - g.each_col() %= (1.0 / arma::sqrt(variance + eps)); + g = (norm.each_col() % stdInv) + (inputMean.each_col() % + var * 2 / input.n_cols); + g.each_col() += arma::sum(norm.each_col() % -stdInv, 1) + var % + arma::mean(-2 * inputMean, 1) / input.n_cols; } template template void BatchNorm::Gradient( - const arma::Mat&& input, + const arma::Mat&& /* input */, arma::Mat&& error, arma::Mat&& gradient) { gradient.set_size(size + size, 1); - - arma::mat normalized = input.each_col() - arma::mean(input, 1) - / arma::sqrt(arma::var(input, 1, 1) + eps); - gradient.submat(0, 0, gamma.n_elem - 1, 0) = arma::sum(normalized % error, 1); gradient.submat(gamma.n_elem, 0, gradient.n_elem - 1, 0) = - arma::sum(error, 1); + arma::sum(error, 1); } template diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 04552fe775..3095dc497d 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -1344,32 +1344,6 @@ BOOST_AUTO_TEST_CASE(BatchNormTest) CheckMatrices(output, result, 1e-1); result.clear(); - // Backward Pass Test. - arma::mat gy; - gy << 0.8402 << 0.9116 << 0.2778 << arma::endr - << 0.3944 << 0.1976 << 0.5540 << arma::endr - << 0.7831 << 0.3352 << 0.4774 << arma::endr; - - model.Backward(std::move(input), std::move(gy), std::move(output)); - result << -0.0780 << 0.1376 << -0.0596 << arma::endr - << 0.0602 << -0.1317 << 0.0715 << arma::endr - << 0.0835 << -0.1493 << 0.0658 << arma::endr; - - CheckMatrices(output, result, 1e-1); - result.clear(); - - // Gradient Test. - model.Gradient(std::move(input), std::move(gy), std::move(output)); - result << 3.4003 << arma::endr - << 0.8183 << arma::endr - << 1.8574 << arma::endr - << 2.0296 << arma::endr - << 1.1460 << arma::endr - << 1.5957 << arma::endr; - - CheckMatrices(output, result, 1e-1); - result.clear(); - // Deterministic Forward Pass test. output = model.TrainingMean(); result << 3.33333333 << arma::endr @@ -1438,7 +1412,7 @@ BOOST_AUTO_TEST_CASE(GradientBatchNormLayerTest) arma::mat input, target; } function; - BOOST_REQUIRE_LE(CheckGradient(function), 1e-3); + BOOST_REQUIRE_LE(CheckGradient(function), 1e-4); } BOOST_AUTO_TEST_SUITE_END();