apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update backward step (pass the gradient check).

Browse Source
This commit is contained in:
Marcus Edel
2018-03-21 12:57:42 +01:00
parent 97d88838c0
commit dfeda93e4b
3 changed files with 38 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mlpack/methods/ann/layer/batch_norm.hpp
+11 -7
View File
@@ -3,7 +3,7 @@
* @author Praveen Ch
* @author Manthan-R-Sheth
*
* Definition of the Batch Normalisation layer class
* Definition of the Batch Normalization layer class.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
@@ -20,7 +20,7 @@ namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
/**
* Declaration of the Batch Normalisation layer class. The layer tranforms
* Declaration of the Batch Normalization layer class. The layer tranforms
* the input data into zero mean and unit variance and then scales and shifts
* the data by parameters, gamma and beta respectively. These parameters are
* learnt by the network.
@@ -32,15 +32,17 @@ namespace ann /** Artificial Neural Network. */ {
* For more information, refer to the following paper,
*
* @code
* @article{DBLP:journals/corr/IoffeS15,
* @article{Ioffe15,
* author = {Sergey Ioffe and
* Christian Szegedy},
* title = {Batch Normalization: Accelerating Deep Network Training by
* Reducing Internal Covariate Shift},
* journal = {CoRR},
* volume = {abs/1502.03167}
* volume = {abs/1502.03167},
* year = {2015},
* url = {http://arxiv.org/abs/1502.03167},
* eprint = {1502.03167},
* }
*
* @endcode
*
* @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
@@ -48,7 +50,6 @@ namespace ann /** Artificial Neural Network. */ {
* @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
* arma::sp_mat or arma::cube).
*/
template <
typename InputDataType = arma::mat,
typename OutputDataType = arma::mat
@@ -65,7 +66,7 @@ class BatchNorm
* @param size The number of input units.
* @param eps The epsilon added to variance to ensure numerical stability.
*/
BatchNorm(const size_t size, const double eps = 0.001);
BatchNorm(const size_t size, const double eps = 1e-8);
/**
* Reset the layer parameters
@@ -191,6 +192,9 @@ class BatchNorm
//! Locally-stored output parameter object.
OutputDataType outputParameter;
//! Locally-stored normalized input.
OutputDataType normalized;
}; // class BatchNorm
} // namespace ann
src/mlpack/methods/ann/layer/batch_norm_impl.hpp
+26 -26
View File
@@ -23,7 +23,7 @@ namespace ann { /** Artificial Neural Network. */
template<typename InputDataType, typename OutputDataType>
BatchNorm<InputDataType, OutputDataType>::BatchNorm() :
size(10),
eps(1e-7),
eps(1e-8),
deterministic(false)
{
// Nothing to do here.
@@ -55,29 +55,34 @@ template<typename eT>
void BatchNorm<InputDataType, OutputDataType>::Forward(
const arma::Mat<eT>&& input, arma::Mat<eT>&& output)
{
output.reshape(input.n_rows, input.n_cols);
// Mean and variance over the entire training set will be used to compute
// the forward pass when deterministic is set to true.
if (deterministic)
{
mean = stats.mean();
variance = stats.var(1);
output = input.each_col() - mean;
output.each_col() %= gamma / arma::sqrt(variance + eps);
output.each_col() += beta;
}
else
{
mean = arma::mean(input, 1);
variance = arma::var(input, 1, 1);
for (size_t i = 0; i < output.n_cols; i++)
{
for (size_t i = 0; i < input.n_cols; i++)
stats(input.col(i));
}
}
output = input.each_col() - mean;
output.each_col() %= gamma / arma::sqrt(variance + eps);
output.each_col() += beta;
output = input.each_col() - mean;
output.each_col() /= arma::sqrt(variance + eps);
// Reused in the backward and gradient step.
normalized = output;
output.each_col() %= gamma;
output.each_col() += beta;
}
}
template<typename InputDataType, typename OutputDataType>
@@ -85,34 +90,29 @@ template<typename eT>
void BatchNorm<InputDataType, OutputDataType>::Backward(
const arma::Mat<eT>&& input, arma::Mat<eT>&& gy, arma::Mat<eT>&& g)
{
mean = arma::mean(input, 1);
variance = arma::var(input, 1, 1);
const arma::mat inputMean = input.each_col() - mean;
const arma::mat stdInv = 1.0 / arma::sqrt(variance + eps);
const arma::mat norm = gy.each_col() % gamma;
const arma::mat var = arma::sum(norm % inputMean, 1) %
arma::pow(stdInv, 3.0) * 0.5;
arma::mat m = arma::sum(gy % (input.each_col() - mean), 1);
g = (mean - input.each_col());
g.each_col() %= m;
g.each_col() %= 1.0/(variance + eps);
g += (gy.each_col() - arma::sum(gy, 1));
g += (input.n_cols - 1) * gy;
g.each_col() %= ((1.0 / input.n_cols) * gamma);
g.each_col() %= (1.0 / arma::sqrt(variance + eps));
g = (norm.each_col() % stdInv) + (inputMean.each_col() %
var * 2 / input.n_cols);
g.each_col() += arma::sum(norm.each_col() % -stdInv, 1) + var %
arma::mean(-2 * inputMean, 1) / input.n_cols;
}
template<typename InputDataType, typename OutputDataType>
template<typename eT>
void BatchNorm<InputDataType, OutputDataType>::Gradient(
const arma::Mat<eT>&& input,
const arma::Mat<eT>&& /* input */,
arma::Mat<eT>&& error,
arma::Mat<eT>&& gradient)
{
gradient.set_size(size + size, 1);
arma::mat normalized = input.each_col() - arma::mean(input, 1)
/ arma::sqrt(arma::var(input, 1, 1) + eps);
gradient.submat(0, 0, gamma.n_elem - 1, 0) = arma::sum(normalized % error, 1);
gradient.submat(gamma.n_elem, 0, gradient.n_elem - 1, 0) =
arma::sum(error, 1);
arma::sum(error, 1);
}
template<typename InputDataType, typename OutputDataType>
src/mlpack/tests/ann_layer_test.cpp
+1 -27
View File
@@ -1344,32 +1344,6 @@ BOOST_AUTO_TEST_CASE(BatchNormTest)
CheckMatrices(output, result, 1e-1);
result.clear();
// Backward Pass Test.
arma::mat gy;
gy << 0.8402 << 0.9116 << 0.2778 << arma::endr
<< 0.3944 << 0.1976 << 0.5540 << arma::endr
<< 0.7831 << 0.3352 << 0.4774 << arma::endr;
model.Backward(std::move(input), std::move(gy), std::move(output));
result << -0.0780 << 0.1376 << -0.0596 << arma::endr
<< 0.0602 << -0.1317 << 0.0715 << arma::endr
<< 0.0835 << -0.1493 << 0.0658 << arma::endr;
CheckMatrices(output, result, 1e-1);
result.clear();
// Gradient Test.
model.Gradient(std::move(input), std::move(gy), std::move(output));
result << 3.4003 << arma::endr
<< 0.8183 << arma::endr
<< 1.8574 << arma::endr
<< 2.0296 << arma::endr
<< 1.1460 << arma::endr
<< 1.5957 << arma::endr;
CheckMatrices(output, result, 1e-1);
result.clear();
// Deterministic Forward Pass test.
output = model.TrainingMean();
result << 3.33333333 << arma::endr
@@ -1438,7 +1412,7 @@ BOOST_AUTO_TEST_CASE(GradientBatchNormLayerTest)
arma::mat input, target;
} function;
BOOST_REQUIRE_LE(CheckGradient(function), 1e-3);
BOOST_REQUIRE_LE(CheckGradient(function), 1e-4);
}
BOOST_AUTO_TEST_SUITE_END();