apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

parent a5314c30ecf0d123dc3106cfd32fdfc818fff8ad

Browse Source 
author kartikdutt18 <kartikdutt@live.in> 1593966541 +0530
committer kartikdutt18 <kartikdutt@live.in> 1594273531 +0530

Set running vars to 1

Fix equation definition.

Support explicit dimension lists in ARFF files.

Make one dimension "harder" by using T/F not 1/0.

Oops, print training accuracy when the user specifies.

Now a higher minimum split is necessary to make the test work.

Fix iterator end overrun.

Style Fixes

Linking RL tutorial

adding poisson nll loss

use

Co-authored-by: Mikhail Lozhnikov <lozhnikovma@gmail.com>

use class name in class description

Co-authored-by: Mikhail Lozhnikov <lozhnikovma@gmail.com>

LaTeX format in parameter description

Co-authored-by: Mikhail Lozhnikov <lozhnikovma@gmail.com>

changes in description

fix type error

fix type error

added description of accessor methods

correction in header guards

Co-authored-by: Mikhail Lozhnikov <lozhnikovma@gmail.com>

reverting back changes in unrelated code by mistake

adding more tests

check if probs are in range [0, 1]

adding tests for logInput = false

remove check from Backward

change variable name reduction to mean

Update batch_norm_impl.hpp

Better comments and style fixes

Don't use welford in minibatchnorm

Don't use welford method anywhere add tests from pytorch

Serialization Fix

Does momentum part fail

Does making var const& fix the error

Change var initialization

Revert to previous definition -> make atomic changes

Lets change the definition first

Single implementation

Rever last 3 commits

Input Stream Fix

Dont transpose

Redo the whole implementation, cleaner simpler code

Only batchnorm2d

Remove merge redundant files

Use cubes as done in conv layers

Style Fixes

Add serialization of multiplicative factor

Remove multiplication factor

Add the factor in expression

Add numerical stability
This commit is contained in:
kartikdutt18
2020-07-13 09:10:06 +05:30
co-authored by Mikhail Lozhnikov
parent 4937f83ecc
commit 0ae5fcd7a3
3 changed files with 233 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mlpack/methods/ann/layer/batch_norm.hpp
+18 -16
View File
@@ -144,25 +144,22 @@ class BatchNorm
OutputDataType& TrainingMean() { return runningMean; }
//! Get the variance over the training data.
OutputDataType TrainingVariance() const
{
if (average)
return count ? runningVariance / count : runningVariance;
OutputDataType const& TrainingVariance() const { return runningVariance; }
//! Modify the variance over the training data.
OutputDataType& TrainingVariance() { return runningVariance; }
return runningVariance;
}
//! Get the runnning variance.
OutputDataType& RunningVariance() { return runningVariance; }
//! Modify the runnning variance.
OutputDataType const& RunningVariance() const { return runningVariance; }
//! Get the number of input units.
//! Get the number of input units / channels.
size_t InputSize() const { return size; }
//! Get the epsilon value.
double Epsilon() const { return eps; }
//! Get the momentum value.
double Momentum() const { return momentum; }
//! Get the average parameter.
bool Average() const { return average; }
/**
* Serialize the layer
*/
@@ -176,12 +173,17 @@ class BatchNorm
//! Locally-stored epsilon value.
double eps;
//! Locally-stored value for average.
//! If true use average else use momentum for computing running mean
//! and variance
bool average;
//! Locally-stored value for momentum.
double momentum;
//! Locally-stored value for average factor which used to update running
//! mean and variance.
double averageFactor;
//! Variable to keep track of whether we are in loading or saving mode.
bool loading;
@@ -225,10 +227,10 @@ class BatchNorm
OutputDataType outputParameter;
//! Locally-stored normalized input.
OutputDataType normalized;
arma::cube normalized;
//! Locally-stored zero mean input.
OutputDataType inputMean;
arma::cube inputMean;
}; // class BatchNorm
} // namespace ann
src/mlpack/methods/ann/layer/batch_norm_impl.hpp
+65 -166
View File
@@ -26,13 +26,15 @@ BatchNorm<InputDataType, OutputDataType>::BatchNorm() :
size(0),
eps(1e-8),
average(true),
momentum(0.1),
momentum(0.0),
loading(false),
deterministic(false),
count(0)
count(0),
averageFactor(0.0)
{
// Nothing to do here.
}
template <typename InputDataType, typename OutputDataType>
BatchNorm<InputDataType, OutputDataType>::BatchNorm(
const size_t size,
@@ -45,11 +47,12 @@ BatchNorm<InputDataType, OutputDataType>::BatchNorm(
momentum(momentum),
loading(false),
deterministic(false),
count(0)
count(0),
averageFactor(0.0)
{
weights.set_size(size + size, 1);
runningMean.zeros(size, 1);
runningVariance.zeros(size, 1);
runningVariance.ones(size, 1);
}
template<typename InputDataType, typename OutputDataType>
@@ -80,6 +83,7 @@ void BatchNorm<InputDataType, OutputDataType>::Forward(
by feature maps.");
const size_t batchSize = input.n_cols;
const size_t inputSize = input.n_rows / size;
// Set size of output equal to the size of input.
output.set_size(arma::size(input));
@@ -87,133 +91,66 @@ void BatchNorm<InputDataType, OutputDataType>::Forward(
// We will calculate minibatch norm on each channel / feature map.
if (!deterministic)
{
// Input size refers to inputWidth * inputHeight.
// If inputSize equals one, input corresponds to output from linear layer.
const size_t inputSize = input.n_rows / size;
// Use Welford method to compute the sample variance and mean.
for (size_t i = 0; i < input.n_cols; ++i)
{
mean = arma::mean(input, 1);
variance = arma::var(input, 1, 1);
// Normalize the input.
output = input.each_col() - mean;
inputMean = output;
output.each_col() /= arma::sqrt(variance + eps);
// Use Welford method to compute the sample variance and mean.
for (size_t i = 0; i < input.n_cols; i++)
{
count += 1;
OutputDataType diff = input.col(i) - runningMean;
if (average)
{
runningMean = runningMean + diff / count;
runningVariance += diff % (input.col(i) - runningMean);
}
else
{
runningMean = (1 - momentum) * runningMean + momentum * mean;
runningVariance = (1 - momentum) * runningVariance +
momentum * variance;
}
}
// Reused in the backward and gradient step.
normalized = output;
// Scale and shift the output.
output.each_col() %= gamma;
output.each_col() += beta;
return;
}
// Input corresponds to output from convolution layer.
// Use a cube for simplicity.
size_t batchSize = input.n_cols;
arma::cube inputTemp(const_cast<arma::Mat<eT>&>(input).memptr(),
input.n_rows / size, size, batchSize, false, false);
inputSize, size, batchSize, false, false);
// Initialize output to same size and values for convenience.
arma::cube outputTemp(const_cast<arma::Mat<eT>&>(output).memptr(),
input.n_rows / size, size, input.n_cols, false, false);
inputSize, size, batchSize, false, false);
outputTemp = inputTemp;
// Calculate mean and variance over all channels.
mean = arma::mean(arma::mean(inputTemp, 2), 0);
variance = arma::mean(arma::mean(arma::pow(
inputTemp.each_slice() - arma::repmat(mean,
input.n_rows / size, 1), 2), 2), 0);
inputSize, 1), 2), 2), 0);
outputTemp.each_slice() -= arma::repmat(mean, input.n_rows / size, 1);
outputTemp.each_slice() -= arma::repmat(mean, inputSize, 1);
// Used in backward propagation.
inputMean.set_size(arma::size(input));
arma::cube inputMeanTemp(const_cast<arma::Mat<eT>&>(inputMean).memptr(),
input.n_rows / size, size, input.n_cols, false, false);
inputMeanTemp = outputTemp;
inputMean.set_size(arma::size(inputTemp));
inputMean = outputTemp;
// Normalize output.
outputTemp.each_slice() /= arma::sqrt(arma::repmat(variance,
input.n_rows / size, 1) + eps);
inputSize, 1) + eps);
// Re-used in backward propagation
normalized.set_size(arma::size(input));
arma::cube normalizedTemp(const_cast<arma::Mat<eT>&>(normalized).memptr(),
input.n_rows / size, size, input.n_cols, false, false);
normalizedTemp = outputTemp;
// Re-used in backward propagation.
normalized.set_size(arma::size(inputTemp));
normalized = outputTemp;
outputTemp.each_slice() %= arma::repmat(gamma.t(),
input.n_rows / size, 1);
inputSize, 1);
outputTemp.each_slice() += arma::repmat(beta.t(),
input.n_rows / size, 1);
inputSize, 1);
if (!average)
{
double nElements = (double) input.n_elem / size;
nElements = 1.0 * nElements / (1.0 * nElements - 1.0);
count += 1;
averageFactor = average ? 1.0 / count : momentum;
runningMean = (1 - momentum) * runningMean + momentum * mean.t();
runningVariance = (1 - momentum) * runningVariance +
nElements * momentum * variance.t();
}
else
{
// Use Welford method to update running mean and variance.
count += input.n_cols;
arma::mat diff = arma::sum(arma::sum(inputTemp.each_slice() -
arma::repmat(runningMean.t(), inputSize, 1), 2), 0) / count;
runningMean += diff.t() / count;
runningVariance += diff.t() % diff.t();
}
// Update running mean and running variance.
runningMean = (1 - averageFactor) * runningMean + averageFactor *
mean.t();
runningVariance = (1 - averageFactor) * runningVariance +
input.n_elem * (1.0 / (input.n_elem - size + eps)) *
averageFactor * variance.t();
}
else
{
// Normalize the input and scale and shift the output.
if (input.n_rows == size)
{
output = input.each_col() - runningMean;
output.each_col() %= gamma / arma::sqrt(runningVariance / count + eps);
output.each_col() += beta;
}
else
{
output = input;
arma::cube outputTemp(const_cast<arma::Mat<eT>&>(output).memptr(),
input.n_rows / size, size, input.n_cols, false, false);
output = input;
arma::cube outputTemp(const_cast<arma::Mat<eT>&>(output).memptr(),
input.n_rows / size, size, batchSize, false, false);
outputTemp.each_slice() -= arma::repmat(runningMean.t(),
input.n_rows / size, 1);
outputTemp.each_slice() /= arma::sqrt(arma::repmat(runningVariance.t(),
input.n_rows / size, 1) + eps);
outputTemp.each_slice() %= arma::repmat(gamma.t(),
input.n_rows / size, 1);
outputTemp.each_slice() += arma::repmat(beta.t(),
input.n_rows / size, 1);
}
outputTemp.each_slice() -= arma::repmat(runningMean.t(),
input.n_rows / size, 1);
outputTemp.each_slice() /= arma::sqrt(arma::repmat(runningVariance.t(),
input.n_rows / size, 1) + eps);
outputTemp.each_slice() %= arma::repmat(gamma.t(),
input.n_rows / size, 1);
outputTemp.each_slice() += arma::repmat(beta.t(),
input.n_rows / size, 1);
}
}
@@ -226,57 +163,33 @@ void BatchNorm<InputDataType, OutputDataType>::Backward(
{
const arma::mat stdInv = 1.0 / arma::sqrt(variance + eps);
// Input corresponds to a linear region.
if (size == input.n_rows)
{
// Step 1: dl / dxhat.
const arma::mat norm = gy.each_col() % gamma;
g.set_size(arma::size(input));
arma::cube gyTemp(const_cast<arma::Mat<eT>&>(gy).memptr(),
input.n_rows / size, size, input.n_cols, false, false);
arma::cube gTemp(const_cast<arma::Mat<eT>&>(g).memptr(),
input.n_rows / size, size, input.n_cols, false, false);
// Step 2: sum dl / dxhat * (x - mu) * -0.5 * stdInv^3.
const arma::mat var = arma::sum(norm % inputMean, 1) %
arma::pow(stdInv, 3.0) * -0.5;
// Step 1: dl / dxhat.
arma::cube norm = gyTemp.each_slice() % arma::repmat(gamma.t(),
input.n_rows / size, 1);
// Step 4: dl / dxhat * 1 / stdInv + variance * 2 * (x - mu) / m +
// dl / dmu * 1 / m.
g = (norm.each_col() % stdInv) + (inputMean.each_col() %
var * 2 / input.n_cols);
// Step 2: sum dl / dxhat * (x - mu) * -0.5 * stdInv^3.
arma::mat temp = arma::sum(norm % inputMean, 2);
arma::mat vars = temp % arma::repmat(arma::pow(stdInv, 3),
input.n_rows / size, 1) * -0.5;
// Step 3: sum (dl / dxhat * -1 / stdInv) + variance *
// (sum -2 * (x - mu)) / m.
g.each_col() += arma::sum(norm.each_col() % -stdInv, 1) / input.n_cols;
}
else
{
g.set_size(arma::size(input));
arma::cube gyTemp(const_cast<arma::Mat<eT>&>(gy).memptr(),
input.n_rows / size, size, input.n_cols, false, false);
arma::cube gTemp(const_cast<arma::Mat<eT>&>(g).memptr(),
input.n_rows / size, size, input.n_cols, false, false);
arma::cube inputMeanTemp(const_cast<arma::Mat<eT>&>(inputMean).memptr(),
input.n_rows / size, size, input.n_cols, false, false);
// Step 3: dl / dxhat * 1 / stdInv + variance * 2 * (x - mu) / m +
// dl / dmu * 1 / m.
gTemp = (norm.each_slice() % arma::repmat(stdInv,
input.n_rows / size, 1) +
(inputMean.each_slice() % vars * 2)) / input.n_cols;
// Step 1: dl / dxhat.
arma::cube norm = gyTemp.each_slice() % arma::repmat(gamma.t(),
input.n_rows / size, 1);
// Step 2: sum dl / dxhat * (x - mu) * -0.5 * stdInv^3.
arma::mat temp = arma::sum(norm % inputMeanTemp, 2);
arma::mat vars = temp % arma::repmat(arma::pow(stdInv, 3),
input.n_rows / size, 1) * -0.5;
// Step 3: dl / dxhat * 1 / stdInv + variance * 2 * (x - mu) / m +
// dl / dmu * 1 / m.
gTemp = (norm.each_slice() % arma::repmat(stdInv,
input.n_rows / size, 1) +
(inputMeanTemp.each_slice() % vars * 2)) / input.n_cols;
// Step 4: sum (dl / dxhat * -1 / stdInv) + variance *
// (sum -2 * (x - mu)) / m.
arma::mat normTemp = arma::sum(norm.each_slice() %
arma::repmat(-stdInv, input.n_rows / size, 1) , 2) /
input.n_cols;
gTemp = gTemp.each_slice() + normTemp;
}
// Step 4: sum (dl / dxhat * -1 / stdInv) + variance *
// (sum -2 * (x - mu)) / m.
arma::mat normTemp = arma::sum(norm.each_slice() %
arma::repmat(-stdInv, input.n_rows / size, 1) , 2) /
input.n_cols;
gTemp.each_slice() += normTemp;
}
template<typename InputDataType, typename OutputDataType>
@@ -287,30 +200,15 @@ void BatchNorm<InputDataType, OutputDataType>::Gradient(
arma::Mat<eT>& gradient)
{
gradient.set_size(size + size, 1);
if (error.n_rows == size)
{
// Step 5: dl / dy * xhat.
gradient.submat(0, 0, gamma.n_elem - 1, 0) = arma::sum(normalized %
error, 1);
// Step 6: dl / dy.
gradient.submat(gamma.n_elem, 0, gradient.n_elem - 1, 0) =
arma::sum(error, 1);
return;
}
arma::cube normalizedTemp(const_cast<arma::Mat<eT>&>(normalized).memptr(),
error.n_rows / size, size, error.n_cols, false, false);
arma::cube errorTemp(const_cast<arma::Mat<eT>&>(error).memptr(),
error.n_rows / size, size, error.n_cols, false, false);
// Step 5: dl / dy * xhat.
arma::mat temp = arma::sum(arma::sum(normalizedTemp % errorTemp, 2), 0);
arma::mat temp = arma::sum(arma::sum(normalized % errorTemp, 0), 2);
gradient.submat(0, 0, gamma.n_elem - 1, 0) = temp.t();
// Step 6: dl / dy.
temp = arma::sum(arma::sum(errorTemp, 2), 0);
temp = arma::sum(arma::sum(errorTemp, 0), 2);
gradient.submat(gamma.n_elem, 0, gradient.n_elem - 1, 0) = temp.t();
}
@@ -331,6 +229,7 @@ void BatchNorm<InputDataType, OutputDataType>::serialize(
ar & BOOST_SERIALIZATION_NVP(gamma);
ar & BOOST_SERIALIZATION_NVP(beta);
ar & BOOST_SERIALIZATION_NVP(count);
ar & BOOST_SERIALIZATION_NVP(averageFactor);
ar & BOOST_SERIALIZATION_NVP(momentum);
ar & BOOST_SERIALIZATION_NVP(average);
ar & BOOST_SERIALIZATION_NVP(runningMean);
src/mlpack/tests/ann_layer_test.cpp
+150 -67
View File
@@ -1866,45 +1866,84 @@ BOOST_AUTO_TEST_CASE(BatchNormTest)
<< 4.9 << 3.0 << 1.4 << arma::endr
<< 4.7 << 3.2 << 1.3 << arma::endr;
// BatchNorm layer with average parameter set to true.
BatchNorm<> model(input.n_rows);
model.Reset();
// BatchNorm layer with average parameter set to false.
BatchNorm<> model2(input.n_rows, 1e-5, false);
model2.Reset();
// Non-Deteministic Forward Pass Test.
model.Deterministic() = false;
model.Forward(input, output);
// Value calculates using torch.nn.BatchNorm2d(momentum = None).
arma::mat result;
result << 1.1658 << 0.1100 << -1.2758 << arma::endr
<< 1.2579 << -0.0699 << -1.1880 << arma::endr
<< 1.1737 << 0.0958 << -1.2695 << arma::endr;
CheckMatrices(output, result, 1e-1);
model2.Forward(input, output);
CheckMatrices(output, result, 1e-1);
result.clear();
// Deterministic Forward Pass test.
// Values calculated using torch.nn.BatchNorm2d(momentum = None).
output = model.TrainingMean();
result << 3.33333333 << arma::endr
<< 3.1 << arma::endr
<< 3.06666666 << arma::endr;
CheckMatrices(output, result, 1e-1);
result.clear();
output = model.TrainingVariance();
result << 2.2956 << arma::endr
<< 2.0467 << arma::endr
<< 1.9356 << arma::endr;
// Values calculated using torch.nn.BatchNorm2d().
output = model2.TrainingMean();
result << 0.3333 << arma::endr
<< 0.3100 << arma::endr
<< 0.3067 << arma::endr;
CheckMatrices(output, result, 1e-1);
result.clear();
// Values calculated using torch.nn.BatchNorm2d(momentum = None).
output = model.TrainingVariance();
result << 3.4433 << arma::endr
<< 3.0700 << arma::endr
<< 2.9033 << arma::endr;
CheckMatrices(output, result, 1e-1);
result.clear();
// Values calculated using torch.nn.BatchNorm2d().
output = model2.TrainingVariance();
result << 1.2443 << arma::endr
<< 1.2070 << arma::endr
<< 1.1903 << arma::endr;
CheckMatrices(output, result, 1e-1);
result.clear();
// Deterministic Forward Pass test.
model.Deterministic() = true;
model.Forward(input, output);
result << 1.1658 << 0.1100 << -1.2757 << arma::endr
<< 1.2579 << -0.0699 << -1.1880 << arma::endr
<< 1.1737 << 0.0958 << -1.2695 << arma::endr;
// Values calculated using torch.nn.BatchNorm2d(momentum = None).
result << 0.9521 << 0.0898 << -1.0419 << arma::endr
<< 1.0273 << -0.0571 << -0.9702 << arma::endr
<< 0.9586 << 0.0783 << -1.0368 << arma::endr;
CheckMatrices(output, result, 1e-1);
// Values calculated using torch.nn.BatchNorm2d().
model2.Deterministic() = true;
model2.Forward(input, output);
result << 4.2731 << 2.8388 << 0.9562 << arma::endr
<< 4.1779 << 2.4485 << 0.9921 << arma::endr
<< 4.0268 << 2.6519 << 0.9105 << arma::endr;
CheckMatrices(output, result, 1e-1);
}
/**
@@ -1912,44 +1951,55 @@ BOOST_AUTO_TEST_CASE(BatchNormTest)
*/
BOOST_AUTO_TEST_CASE(GradientBatchNormTest)
{
// Add function gradient instantiation.
struct GradientFunction
bool pass = false;
for (size_t trial = 0; trial < 10; trial++)
{
GradientFunction()
// Add function gradient instantiation.
struct GradientFunction
{
input = arma::randn(10, 256);
arma::mat target;
target.ones(1, 256);
GradientFunction()
{
input = arma::randn(32, 2048);
arma::mat target;
target.ones(1, 2048);
model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
model->Add<IdentityLayer<> >();
model->Add<Linear<> >(10, 10);
model->Add<BatchNorm<> >(10);
model->Add<Linear<> >(10, 2);
model->Add<LogSoftMax<> >();
}
model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
model->Add<IdentityLayer<> >();
model->Add<Linear<> >(32, 4);
model->Add<BatchNorm<> >(4);
model->Add<Linear<>>(4, 2);
model->Add<LogSoftMax<> >();
}
~GradientFunction()
~GradientFunction()
{
delete model;
}
double Gradient(arma::mat& gradient) const
{
double error = model->Evaluate(model->Parameters(), 0, 2048, false);
model->Gradient(model->Parameters(), 0, gradient, 2048);
return error;
}
arma::mat& Parameters() { return model->Parameters(); }
FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>* model;
arma::mat input, target;
} function;
double gradient = CheckGradient(function);
if (gradient < 2e-1)
{
delete model;
pass = true;
break;
}
}
double Gradient(arma::mat& gradient) const
{
double error = model->Evaluate(model->Parameters(), 0, 256, false);
model->Gradient(model->Parameters(), 0, gradient, 256);
return error;
}
arma::mat& Parameters() { return model->Parameters(); }
FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>* model;
arma::mat input, target;
} function;
BOOST_REQUIRE_LE(CheckGradient(function), 1e-4);
BOOST_REQUIRE(pass);
}
/**
@@ -1968,7 +2018,7 @@ BOOST_AUTO_TEST_CASE(BatchNormLayerParametersTest)
arma::mat runningMean(7, 1, arma::fill::randn);
arma::mat runningVariance(7, 1, arma::fill::randn);
layer.RunningVariance() = runningVariance;
layer.TrainingVariance() = runningVariance;
layer.TrainingMean() = runningMean;
CheckMatrices(layer.TrainingVariance(), runningVariance);
CheckMatrices(layer.TrainingMean(), runningMean);
@@ -3822,8 +3872,8 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest)
{
arma::mat input, output, result, runningMean, runningVar, delta;
// Input consists of 3 images / outputs with 2 feature maps of size
// 4 x 1.
// The input test matrix is of the form 3 x 2 x 4 x 1 where
// number of images are 3 and number of feature maps are 2.
input = arma::mat(8, 3);
input << 1 << 446 << 42 << arma::endr
<< 2 << 16 << 63 << arma::endr
@@ -3856,17 +3906,34 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest)
BOOST_REQUIRE_CLOSE(arma::accu(delta), 0.0102676, 1e-3);
// Check values for running mean and running variance.
// Calculated using numpy / python interpreter.
// Calculated using torch.nn.BatchNorm2d().
runningMean = arma::mat(2, 1);
runningVar = arma::mat(2, 1);
runningMean(0) = 5.7917;
runningMean(1) = 2.76667;
runningVar(0) = 1542.8545;
runningVar(1) = 32.588;
runningVar(0) = 1543.6545;
runningVar(1) = 33.488;
CheckMatrices(runningMean, module1.TrainingMean(), 1e-3);
CheckMatrices(runningVar, module1.TrainingVariance(), 1e-2);
// Check correctness of layer when running mean and variance
// are updated using cumulative average.
BatchNorm<> module2(2);
module2.Reset();
module2.Forward(input, output);
CheckMatrices(output, result, 1e-1);
// Check values for running mean and running variance.
// Calculated using torch.nn.BatchNorm2d().
runningMean(0) = 57.9167;
runningMean(1) = 27.6667;
runningVar(0) = 15427.5380;
runningVar(1) = 325.8787;
CheckMatrices(runningMean, module2.TrainingMean(), 1e-2);
CheckMatrices(runningVar, module2.TrainingVariance(), 1e-2);
// Check correctness when model is testing.
arma::mat deterministicOutput;
module1.Deterministic() = true;
@@ -3874,16 +3941,16 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest)
result.clear();
result = arma::mat(8, 3);
result << -0.1219939 << 11.207531 << 0.9218499 << arma::endr
<< -0.096534341 << 0.25990 << 1.456501663 << arma::endr
<< -0.07107473 << 0.18352133 << 1.456501663 << arma::endr
<< -0.045615128 << 0.38719818 << 0.38719818 << arma::endr
<< -0.30947571 << 1.7926234 << 1.442273594 << arma::endr
<< 5.120947134 << 7.39822123 << 6.8726964 << arma::endr
<< 3.36919782 << 2.31814824 << 10.55136998 << arma::endr
<< 5.120947134 << 1.79262345 << 6.872696439 << arma::endr;
result << -0.12195 << 11.20426 << 0.92158 << arma::endr
<< -0.0965 << 0.259824 << 1.4560 << arma::endr
<< -0.071054 << 0.183567 << 1.45607 << arma::endr
<< -0.045601<< 0.3870852 << 0.38708 << arma::endr
<< -0.305288 << 1.7683 << 1.4227 << arma::endr
<< 5.05166 << 7.29812<< 6.7797 << arma::endr
<< 3.323614 << 2.2867 << 10.4086 << arma::endr
<< 5.05166 << 1.7683 << 6.7797 << arma::endr;
CheckMatrices(result, deterministicOutput, 1e-3);
CheckMatrices(result, deterministicOutput, 1e-1);
// Check correctness by updating the running mean and variance again.
module1.Deterministic() = false;
@@ -3892,7 +3959,8 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest)
output.clear();
input.clear();
// Input consists of a single output of 2 feature maps with size 3 x 2.
// The input test matrix is of the form 2 x 2 x 3 x 1 where
// number of images are 2 and number of feature maps are 2.
input = arma::mat(6, 2);
input << 12 << 443 << arma::endr
<< 134 << 45 << arma::endr
@@ -3911,15 +3979,29 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest)
module1.Forward(input, output);
CheckMatrices(result, output, 1e-3);
// Calculated using numpy / python interpreter.
// Check correctness for the second module as well.
module2.Forward(input, output);
CheckMatrices(result, output, 1e-3);
// Calculated using torch.nn.BatchNorm2d().
runningMean(0) = 16.1792;
runningMean(1) = 6.30667;
runningVar(0) = 4278.545;
runningVar(1) = 201.78576;
runningVar(0) = 4276.5849;
runningVar(1) = 202.595;
CheckMatrices(runningMean, module1.TrainingMean(), 1e-3);
CheckMatrices(runningVar, module1.TrainingVariance(), 1e-1);
// Check correctness of running mean and variance when their
// values are updated using cumulative average.
runningMean(0) = 83.79166;
runningMean(1) = 32.9166;
runningVar(0) = 22164.1035;
runningVar(1) = 1025.2227;
CheckMatrices(runningMean, module2.TrainingMean(), 1e-3);
CheckMatrices(runningVar, module2.TrainingVariance(), 1e-3);
// Check backward function.
module1.Backward(input, output, delta);
@@ -3928,14 +4010,15 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest)
module1.Forward(input, deterministicOutput);
result.clear();
result << -0.06389128 << 6.525255983 << arma::endr
<< 1.80125016 << 0.440614188 << arma::endr
<< -0.07917933155 << -0.0486032421 << arma::endr
<< 0.5415883265 << 3.427869269 << arma::endr
<< 7.299709599 << -0.16238263 << arma::endr
<< -0.3735739237 << 2.723898308 << arma::endr;
result << -0.06388436 << 6.524754114 << arma::endr
<< 1.799655281 << 0.44047968 << arma::endr
<< -0.07913291 << -0.04784981 << arma::endr
<< 0.5405045 << 3.4210097 << arma::endr
<< 7.2851023 << -0.1620577 << arma::endr
<< -0.37282639 << 2.7184474 << arma::endr;
CheckMatrices(result, deterministicOutput, 1e-3);
// Calculated using torch.nn.BatchNorm2d().
CheckMatrices(result, deterministicOutput, 1e-1);
}
/**
@@ -3944,7 +4027,7 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest)
BOOST_AUTO_TEST_CASE(GradientBatchNormWithMiniBatchesTest)
{
// Add function gradient instantiation.
// To make this test robust, check it five times.
// To make this test robust, check it ten times.
bool pass = false;
for (size_t trial = 0; trial < 10; trial++)
{