From 0ae5fcd7a3b23714ea190ca13a72ae3f96a6729e Mon Sep 17 00:00:00 2001 From: kartikdutt18 Date: Sun, 5 Jul 2020 21:59:01 +0530 Subject: [PATCH] parent a5314c30ecf0d123dc3106cfd32fdfc818fff8ad author kartikdutt18 1593966541 +0530 committer kartikdutt18 1594273531 +0530 Set running vars to 1 Fix equation definition. Support explicit dimension lists in ARFF files. Make one dimension "harder" by using T/F not 1/0. Oops, print training accuracy when the user specifies. Now a higher minimum split is necessary to make the test work. Fix iterator end overrun. Style Fixes Linking RL tutorial adding poisson nll loss use Co-authored-by: Mikhail Lozhnikov use class name in class description Co-authored-by: Mikhail Lozhnikov LaTeX format in parameter description Co-authored-by: Mikhail Lozhnikov changes in description fix type error fix type error added description of accessor methods correction in header guards Co-authored-by: Mikhail Lozhnikov reverting back changes in unrelated code by mistake adding more tests check if probs are in range [0, 1] adding tests for logInput = false remove check from Backward change variable name reduction to mean Update batch_norm_impl.hpp Better comments and style fixes Don't use welford in minibatchnorm Don't use welford method anywhere add tests from pytorch Serialization Fix Does momentum part fail Does making var const& fix the error Change var initialization Revert to previous definition -> make atomic changes Lets change the definition first Single implementation Rever last 3 commits Input Stream Fix Dont transpose Redo the whole implementation, cleaner simpler code Only batchnorm2d Remove merge redundant files Use cubes as done in conv layers Style Fixes Add serialization of multiplicative factor Remove multiplication factor Add the factor in expression Add numerical stability --- src/mlpack/methods/ann/layer/batch_norm.hpp | 34 +-- .../methods/ann/layer/batch_norm_impl.hpp | 231 +++++------------- src/mlpack/tests/ann_layer_test.cpp | 217 +++++++++++----- 3 files changed, 233 insertions(+), 249 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batch_norm.hpp b/src/mlpack/methods/ann/layer/batch_norm.hpp index 7e7c36b249..8c07fd0fc5 100644 --- a/src/mlpack/methods/ann/layer/batch_norm.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm.hpp @@ -144,25 +144,22 @@ class BatchNorm OutputDataType& TrainingMean() { return runningMean; } //! Get the variance over the training data. - OutputDataType TrainingVariance() const - { - if (average) - return count ? runningVariance / count : runningVariance; + OutputDataType const& TrainingVariance() const { return runningVariance; } + //! Modify the variance over the training data. + OutputDataType& TrainingVariance() { return runningVariance; } - return runningVariance; - } - - //! Get the runnning variance. - OutputDataType& RunningVariance() { return runningVariance; } - //! Modify the runnning variance. - OutputDataType const& RunningVariance() const { return runningVariance; } - - //! Get the number of input units. + //! Get the number of input units / channels. size_t InputSize() const { return size; } //! Get the epsilon value. double Epsilon() const { return eps; } + //! Get the momentum value. + double Momentum() const { return momentum; } + + //! Get the average parameter. + bool Average() const { return average; } + /** * Serialize the layer */ @@ -176,12 +173,17 @@ class BatchNorm //! Locally-stored epsilon value. double eps; - //! Locally-stored value for average. + //! If true use average else use momentum for computing running mean + //! and variance bool average; //! Locally-stored value for momentum. double momentum; + //! Locally-stored value for average factor which used to update running + //! mean and variance. + double averageFactor; + //! Variable to keep track of whether we are in loading or saving mode. bool loading; @@ -225,10 +227,10 @@ class BatchNorm OutputDataType outputParameter; //! Locally-stored normalized input. - OutputDataType normalized; + arma::cube normalized; //! Locally-stored zero mean input. - OutputDataType inputMean; + arma::cube inputMean; }; // class BatchNorm } // namespace ann diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index a6c1d3e5c9..11f5ff8a9a 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -26,13 +26,15 @@ BatchNorm::BatchNorm() : size(0), eps(1e-8), average(true), - momentum(0.1), + momentum(0.0), loading(false), deterministic(false), - count(0) + count(0), + averageFactor(0.0) { // Nothing to do here. } + template BatchNorm::BatchNorm( const size_t size, @@ -45,11 +47,12 @@ BatchNorm::BatchNorm( momentum(momentum), loading(false), deterministic(false), - count(0) + count(0), + averageFactor(0.0) { weights.set_size(size + size, 1); runningMean.zeros(size, 1); - runningVariance.zeros(size, 1); + runningVariance.ones(size, 1); } template @@ -80,6 +83,7 @@ void BatchNorm::Forward( by feature maps."); const size_t batchSize = input.n_cols; + const size_t inputSize = input.n_rows / size; // Set size of output equal to the size of input. output.set_size(arma::size(input)); @@ -87,133 +91,66 @@ void BatchNorm::Forward( // We will calculate minibatch norm on each channel / feature map. if (!deterministic) { - // Input size refers to inputWidth * inputHeight. - // If inputSize equals one, input corresponds to output from linear layer. - const size_t inputSize = input.n_rows / size; - - // Use Welford method to compute the sample variance and mean. - for (size_t i = 0; i < input.n_cols; ++i) - { - mean = arma::mean(input, 1); - variance = arma::var(input, 1, 1); - - // Normalize the input. - output = input.each_col() - mean; - inputMean = output; - output.each_col() /= arma::sqrt(variance + eps); - - // Use Welford method to compute the sample variance and mean. - for (size_t i = 0; i < input.n_cols; i++) - { - count += 1; - - OutputDataType diff = input.col(i) - runningMean; - if (average) - { - runningMean = runningMean + diff / count; - runningVariance += diff % (input.col(i) - runningMean); - } - else - { - runningMean = (1 - momentum) * runningMean + momentum * mean; - runningVariance = (1 - momentum) * runningVariance + - momentum * variance; - } - } - - // Reused in the backward and gradient step. - normalized = output; - - // Scale and shift the output. - output.each_col() %= gamma; - output.each_col() += beta; - return; - } - // Input corresponds to output from convolution layer. // Use a cube for simplicity. - size_t batchSize = input.n_cols; arma::cube inputTemp(const_cast&>(input).memptr(), - input.n_rows / size, size, batchSize, false, false); + inputSize, size, batchSize, false, false); // Initialize output to same size and values for convenience. arma::cube outputTemp(const_cast&>(output).memptr(), - input.n_rows / size, size, input.n_cols, false, false); + inputSize, size, batchSize, false, false); outputTemp = inputTemp; // Calculate mean and variance over all channels. mean = arma::mean(arma::mean(inputTemp, 2), 0); variance = arma::mean(arma::mean(arma::pow( inputTemp.each_slice() - arma::repmat(mean, - input.n_rows / size, 1), 2), 2), 0); + inputSize, 1), 2), 2), 0); - outputTemp.each_slice() -= arma::repmat(mean, input.n_rows / size, 1); + outputTemp.each_slice() -= arma::repmat(mean, inputSize, 1); // Used in backward propagation. - inputMean.set_size(arma::size(input)); - - arma::cube inputMeanTemp(const_cast&>(inputMean).memptr(), - input.n_rows / size, size, input.n_cols, false, false); - inputMeanTemp = outputTemp; + inputMean.set_size(arma::size(inputTemp)); + inputMean = outputTemp; // Normalize output. outputTemp.each_slice() /= arma::sqrt(arma::repmat(variance, - input.n_rows / size, 1) + eps); + inputSize, 1) + eps); - // Re-used in backward propagation - normalized.set_size(arma::size(input)); - arma::cube normalizedTemp(const_cast&>(normalized).memptr(), - input.n_rows / size, size, input.n_cols, false, false); - normalizedTemp = outputTemp; + // Re-used in backward propagation. + normalized.set_size(arma::size(inputTemp)); + normalized = outputTemp; outputTemp.each_slice() %= arma::repmat(gamma.t(), - input.n_rows / size, 1); + inputSize, 1); outputTemp.each_slice() += arma::repmat(beta.t(), - input.n_rows / size, 1); + inputSize, 1); - if (!average) - { - double nElements = (double) input.n_elem / size; - nElements = 1.0 * nElements / (1.0 * nElements - 1.0); + count += 1; + averageFactor = average ? 1.0 / count : momentum; - runningMean = (1 - momentum) * runningMean + momentum * mean.t(); - runningVariance = (1 - momentum) * runningVariance + - nElements * momentum * variance.t(); - } - else - { - // Use Welford method to update running mean and variance. - count += input.n_cols; - arma::mat diff = arma::sum(arma::sum(inputTemp.each_slice() - - arma::repmat(runningMean.t(), inputSize, 1), 2), 0) / count; - runningMean += diff.t() / count; - runningVariance += diff.t() % diff.t(); - } + // Update running mean and running variance. + runningMean = (1 - averageFactor) * runningMean + averageFactor * + mean.t(); + runningVariance = (1 - averageFactor) * runningVariance + + input.n_elem * (1.0 / (input.n_elem - size + eps)) * + averageFactor * variance.t(); } else { // Normalize the input and scale and shift the output. - if (input.n_rows == size) - { - output = input.each_col() - runningMean; - output.each_col() %= gamma / arma::sqrt(runningVariance / count + eps); - output.each_col() += beta; - } - else - { - output = input; - arma::cube outputTemp(const_cast&>(output).memptr(), - input.n_rows / size, size, input.n_cols, false, false); + output = input; + arma::cube outputTemp(const_cast&>(output).memptr(), + input.n_rows / size, size, batchSize, false, false); - outputTemp.each_slice() -= arma::repmat(runningMean.t(), - input.n_rows / size, 1); - outputTemp.each_slice() /= arma::sqrt(arma::repmat(runningVariance.t(), - input.n_rows / size, 1) + eps); - outputTemp.each_slice() %= arma::repmat(gamma.t(), - input.n_rows / size, 1); - outputTemp.each_slice() += arma::repmat(beta.t(), - input.n_rows / size, 1); - } + outputTemp.each_slice() -= arma::repmat(runningMean.t(), + input.n_rows / size, 1); + outputTemp.each_slice() /= arma::sqrt(arma::repmat(runningVariance.t(), + input.n_rows / size, 1) + eps); + outputTemp.each_slice() %= arma::repmat(gamma.t(), + input.n_rows / size, 1); + outputTemp.each_slice() += arma::repmat(beta.t(), + input.n_rows / size, 1); } } @@ -226,57 +163,33 @@ void BatchNorm::Backward( { const arma::mat stdInv = 1.0 / arma::sqrt(variance + eps); - // Input corresponds to a linear region. - if (size == input.n_rows) - { - // Step 1: dl / dxhat. - const arma::mat norm = gy.each_col() % gamma; + g.set_size(arma::size(input)); + arma::cube gyTemp(const_cast&>(gy).memptr(), + input.n_rows / size, size, input.n_cols, false, false); + arma::cube gTemp(const_cast&>(g).memptr(), + input.n_rows / size, size, input.n_cols, false, false); - // Step 2: sum dl / dxhat * (x - mu) * -0.5 * stdInv^3. - const arma::mat var = arma::sum(norm % inputMean, 1) % - arma::pow(stdInv, 3.0) * -0.5; + // Step 1: dl / dxhat. + arma::cube norm = gyTemp.each_slice() % arma::repmat(gamma.t(), + input.n_rows / size, 1); - // Step 4: dl / dxhat * 1 / stdInv + variance * 2 * (x - mu) / m + - // dl / dmu * 1 / m. - g = (norm.each_col() % stdInv) + (inputMean.each_col() % - var * 2 / input.n_cols); + // Step 2: sum dl / dxhat * (x - mu) * -0.5 * stdInv^3. + arma::mat temp = arma::sum(norm % inputMean, 2); + arma::mat vars = temp % arma::repmat(arma::pow(stdInv, 3), + input.n_rows / size, 1) * -0.5; - // Step 3: sum (dl / dxhat * -1 / stdInv) + variance * - // (sum -2 * (x - mu)) / m. - g.each_col() += arma::sum(norm.each_col() % -stdInv, 1) / input.n_cols; - } - else - { - g.set_size(arma::size(input)); - arma::cube gyTemp(const_cast&>(gy).memptr(), - input.n_rows / size, size, input.n_cols, false, false); - arma::cube gTemp(const_cast&>(g).memptr(), - input.n_rows / size, size, input.n_cols, false, false); - arma::cube inputMeanTemp(const_cast&>(inputMean).memptr(), - input.n_rows / size, size, input.n_cols, false, false); + // Step 3: dl / dxhat * 1 / stdInv + variance * 2 * (x - mu) / m + + // dl / dmu * 1 / m. + gTemp = (norm.each_slice() % arma::repmat(stdInv, + input.n_rows / size, 1) + + (inputMean.each_slice() % vars * 2)) / input.n_cols; - // Step 1: dl / dxhat. - arma::cube norm = gyTemp.each_slice() % arma::repmat(gamma.t(), - input.n_rows / size, 1); - - // Step 2: sum dl / dxhat * (x - mu) * -0.5 * stdInv^3. - arma::mat temp = arma::sum(norm % inputMeanTemp, 2); - arma::mat vars = temp % arma::repmat(arma::pow(stdInv, 3), - input.n_rows / size, 1) * -0.5; - - // Step 3: dl / dxhat * 1 / stdInv + variance * 2 * (x - mu) / m + - // dl / dmu * 1 / m. - gTemp = (norm.each_slice() % arma::repmat(stdInv, - input.n_rows / size, 1) + - (inputMeanTemp.each_slice() % vars * 2)) / input.n_cols; - - // Step 4: sum (dl / dxhat * -1 / stdInv) + variance * - // (sum -2 * (x - mu)) / m. - arma::mat normTemp = arma::sum(norm.each_slice() % - arma::repmat(-stdInv, input.n_rows / size, 1) , 2) / - input.n_cols; - gTemp = gTemp.each_slice() + normTemp; - } + // Step 4: sum (dl / dxhat * -1 / stdInv) + variance * + // (sum -2 * (x - mu)) / m. + arma::mat normTemp = arma::sum(norm.each_slice() % + arma::repmat(-stdInv, input.n_rows / size, 1) , 2) / + input.n_cols; + gTemp.each_slice() += normTemp; } template @@ -287,30 +200,15 @@ void BatchNorm::Gradient( arma::Mat& gradient) { gradient.set_size(size + size, 1); - - if (error.n_rows == size) - { - // Step 5: dl / dy * xhat. - gradient.submat(0, 0, gamma.n_elem - 1, 0) = arma::sum(normalized % - error, 1); - - // Step 6: dl / dy. - gradient.submat(gamma.n_elem, 0, gradient.n_elem - 1, 0) = - arma::sum(error, 1); - return; - } - - arma::cube normalizedTemp(const_cast&>(normalized).memptr(), - error.n_rows / size, size, error.n_cols, false, false); arma::cube errorTemp(const_cast&>(error).memptr(), error.n_rows / size, size, error.n_cols, false, false); // Step 5: dl / dy * xhat. - arma::mat temp = arma::sum(arma::sum(normalizedTemp % errorTemp, 2), 0); + arma::mat temp = arma::sum(arma::sum(normalized % errorTemp, 0), 2); gradient.submat(0, 0, gamma.n_elem - 1, 0) = temp.t(); // Step 6: dl / dy. - temp = arma::sum(arma::sum(errorTemp, 2), 0); + temp = arma::sum(arma::sum(errorTemp, 0), 2); gradient.submat(gamma.n_elem, 0, gradient.n_elem - 1, 0) = temp.t(); } @@ -331,6 +229,7 @@ void BatchNorm::serialize( ar & BOOST_SERIALIZATION_NVP(gamma); ar & BOOST_SERIALIZATION_NVP(beta); ar & BOOST_SERIALIZATION_NVP(count); + ar & BOOST_SERIALIZATION_NVP(averageFactor); ar & BOOST_SERIALIZATION_NVP(momentum); ar & BOOST_SERIALIZATION_NVP(average); ar & BOOST_SERIALIZATION_NVP(runningMean); diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index e3b2b2018b..8cd06e56f5 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -1866,45 +1866,84 @@ BOOST_AUTO_TEST_CASE(BatchNormTest) << 4.9 << 3.0 << 1.4 << arma::endr << 4.7 << 3.2 << 1.3 << arma::endr; + // BatchNorm layer with average parameter set to true. BatchNorm<> model(input.n_rows); model.Reset(); + // BatchNorm layer with average parameter set to false. + BatchNorm<> model2(input.n_rows, 1e-5, false); + model2.Reset(); + // Non-Deteministic Forward Pass Test. model.Deterministic() = false; model.Forward(input, output); + + // Value calculates using torch.nn.BatchNorm2d(momentum = None). arma::mat result; result << 1.1658 << 0.1100 << -1.2758 << arma::endr << 1.2579 << -0.0699 << -1.1880 << arma::endr << 1.1737 << 0.0958 << -1.2695 << arma::endr; CheckMatrices(output, result, 1e-1); + + model2.Forward(input, output); + CheckMatrices(output, result, 1e-1); result.clear(); - // Deterministic Forward Pass test. + // Values calculated using torch.nn.BatchNorm2d(momentum = None). output = model.TrainingMean(); result << 3.33333333 << arma::endr << 3.1 << arma::endr << 3.06666666 << arma::endr; CheckMatrices(output, result, 1e-1); - result.clear(); - output = model.TrainingVariance(); - result << 2.2956 << arma::endr - << 2.0467 << arma::endr - << 1.9356 << arma::endr; + // Values calculated using torch.nn.BatchNorm2d(). + output = model2.TrainingMean(); + result << 0.3333 << arma::endr + << 0.3100 << arma::endr + << 0.3067 << arma::endr; CheckMatrices(output, result, 1e-1); result.clear(); + // Values calculated using torch.nn.BatchNorm2d(momentum = None). + output = model.TrainingVariance(); + result << 3.4433 << arma::endr + << 3.0700 << arma::endr + << 2.9033 << arma::endr; + + CheckMatrices(output, result, 1e-1); + result.clear(); + + // Values calculated using torch.nn.BatchNorm2d(). + output = model2.TrainingVariance(); + result << 1.2443 << arma::endr + << 1.2070 << arma::endr + << 1.1903 << arma::endr; + + CheckMatrices(output, result, 1e-1); + result.clear(); + + // Deterministic Forward Pass test. model.Deterministic() = true; model.Forward(input, output); - result << 1.1658 << 0.1100 << -1.2757 << arma::endr - << 1.2579 << -0.0699 << -1.1880 << arma::endr - << 1.1737 << 0.0958 << -1.2695 << arma::endr; + // Values calculated using torch.nn.BatchNorm2d(momentum = None). + result << 0.9521 << 0.0898 << -1.0419 << arma::endr + << 1.0273 << -0.0571 << -0.9702 << arma::endr + << 0.9586 << 0.0783 << -1.0368 << arma::endr; CheckMatrices(output, result, 1e-1); + + // Values calculated using torch.nn.BatchNorm2d(). + model2.Deterministic() = true; + model2.Forward(input, output); + + result << 4.2731 << 2.8388 << 0.9562 << arma::endr + << 4.1779 << 2.4485 << 0.9921 << arma::endr + << 4.0268 << 2.6519 << 0.9105 << arma::endr; + CheckMatrices(output, result, 1e-1); } /** @@ -1912,44 +1951,55 @@ BOOST_AUTO_TEST_CASE(BatchNormTest) */ BOOST_AUTO_TEST_CASE(GradientBatchNormTest) { - // Add function gradient instantiation. - struct GradientFunction + bool pass = false; + for (size_t trial = 0; trial < 10; trial++) { - GradientFunction() + // Add function gradient instantiation. + struct GradientFunction { - input = arma::randn(10, 256); - arma::mat target; - target.ones(1, 256); + GradientFunction() + { + input = arma::randn(32, 2048); + arma::mat target; + target.ones(1, 2048); - model = new FFN, NguyenWidrowInitialization>(); - model->Predictors() = input; - model->Responses() = target; - model->Add >(); - model->Add >(10, 10); - model->Add >(10); - model->Add >(10, 2); - model->Add >(); - } + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(32, 4); + model->Add >(4); + model->Add>(4, 2); + model->Add >(); + } - ~GradientFunction() + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 2048, false); + model->Gradient(model->Parameters(), 0, gradient, 2048); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN, NguyenWidrowInitialization>* model; + arma::mat input, target; + } function; + + double gradient = CheckGradient(function); + if (gradient < 2e-1) { - delete model; + pass = true; + break; } + } - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 256, false); - model->Gradient(model->Parameters(), 0, gradient, 256); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN, NguyenWidrowInitialization>* model; - arma::mat input, target; - } function; - - BOOST_REQUIRE_LE(CheckGradient(function), 1e-4); + BOOST_REQUIRE(pass); } /** @@ -1968,7 +2018,7 @@ BOOST_AUTO_TEST_CASE(BatchNormLayerParametersTest) arma::mat runningMean(7, 1, arma::fill::randn); arma::mat runningVariance(7, 1, arma::fill::randn); - layer.RunningVariance() = runningVariance; + layer.TrainingVariance() = runningVariance; layer.TrainingMean() = runningMean; CheckMatrices(layer.TrainingVariance(), runningVariance); CheckMatrices(layer.TrainingMean(), runningMean); @@ -3822,8 +3872,8 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest) { arma::mat input, output, result, runningMean, runningVar, delta; - // Input consists of 3 images / outputs with 2 feature maps of size - // 4 x 1. + // The input test matrix is of the form 3 x 2 x 4 x 1 where + // number of images are 3 and number of feature maps are 2. input = arma::mat(8, 3); input << 1 << 446 << 42 << arma::endr << 2 << 16 << 63 << arma::endr @@ -3856,17 +3906,34 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest) BOOST_REQUIRE_CLOSE(arma::accu(delta), 0.0102676, 1e-3); // Check values for running mean and running variance. - // Calculated using numpy / python interpreter. + // Calculated using torch.nn.BatchNorm2d(). runningMean = arma::mat(2, 1); runningVar = arma::mat(2, 1); runningMean(0) = 5.7917; runningMean(1) = 2.76667; - runningVar(0) = 1542.8545; - runningVar(1) = 32.588; + runningVar(0) = 1543.6545; + runningVar(1) = 33.488; CheckMatrices(runningMean, module1.TrainingMean(), 1e-3); CheckMatrices(runningVar, module1.TrainingVariance(), 1e-2); + // Check correctness of layer when running mean and variance + // are updated using cumulative average. + BatchNorm<> module2(2); + module2.Reset(); + module2.Forward(input, output); + CheckMatrices(output, result, 1e-1); + + // Check values for running mean and running variance. + // Calculated using torch.nn.BatchNorm2d(). + runningMean(0) = 57.9167; + runningMean(1) = 27.6667; + runningVar(0) = 15427.5380; + runningVar(1) = 325.8787; + + CheckMatrices(runningMean, module2.TrainingMean(), 1e-2); + CheckMatrices(runningVar, module2.TrainingVariance(), 1e-2); + // Check correctness when model is testing. arma::mat deterministicOutput; module1.Deterministic() = true; @@ -3874,16 +3941,16 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest) result.clear(); result = arma::mat(8, 3); - result << -0.1219939 << 11.207531 << 0.9218499 << arma::endr - << -0.096534341 << 0.25990 << 1.456501663 << arma::endr - << -0.07107473 << 0.18352133 << 1.456501663 << arma::endr - << -0.045615128 << 0.38719818 << 0.38719818 << arma::endr - << -0.30947571 << 1.7926234 << 1.442273594 << arma::endr - << 5.120947134 << 7.39822123 << 6.8726964 << arma::endr - << 3.36919782 << 2.31814824 << 10.55136998 << arma::endr - << 5.120947134 << 1.79262345 << 6.872696439 << arma::endr; + result << -0.12195 << 11.20426 << 0.92158 << arma::endr + << -0.0965 << 0.259824 << 1.4560 << arma::endr + << -0.071054 << 0.183567 << 1.45607 << arma::endr + << -0.045601<< 0.3870852 << 0.38708 << arma::endr + << -0.305288 << 1.7683 << 1.4227 << arma::endr + << 5.05166 << 7.29812<< 6.7797 << arma::endr + << 3.323614 << 2.2867 << 10.4086 << arma::endr + << 5.05166 << 1.7683 << 6.7797 << arma::endr; - CheckMatrices(result, deterministicOutput, 1e-3); + CheckMatrices(result, deterministicOutput, 1e-1); // Check correctness by updating the running mean and variance again. module1.Deterministic() = false; @@ -3892,7 +3959,8 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest) output.clear(); input.clear(); - // Input consists of a single output of 2 feature maps with size 3 x 2. + // The input test matrix is of the form 2 x 2 x 3 x 1 where + // number of images are 2 and number of feature maps are 2. input = arma::mat(6, 2); input << 12 << 443 << arma::endr << 134 << 45 << arma::endr @@ -3911,15 +3979,29 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest) module1.Forward(input, output); CheckMatrices(result, output, 1e-3); - // Calculated using numpy / python interpreter. + // Check correctness for the second module as well. + module2.Forward(input, output); + CheckMatrices(result, output, 1e-3); + + // Calculated using torch.nn.BatchNorm2d(). runningMean(0) = 16.1792; runningMean(1) = 6.30667; - runningVar(0) = 4278.545; - runningVar(1) = 201.78576; + runningVar(0) = 4276.5849; + runningVar(1) = 202.595; CheckMatrices(runningMean, module1.TrainingMean(), 1e-3); CheckMatrices(runningVar, module1.TrainingVariance(), 1e-1); + // Check correctness of running mean and variance when their + // values are updated using cumulative average. + runningMean(0) = 83.79166; + runningMean(1) = 32.9166; + runningVar(0) = 22164.1035; + runningVar(1) = 1025.2227; + + CheckMatrices(runningMean, module2.TrainingMean(), 1e-3); + CheckMatrices(runningVar, module2.TrainingVariance(), 1e-3); + // Check backward function. module1.Backward(input, output, delta); @@ -3928,14 +4010,15 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest) module1.Forward(input, deterministicOutput); result.clear(); - result << -0.06389128 << 6.525255983 << arma::endr - << 1.80125016 << 0.440614188 << arma::endr - << -0.07917933155 << -0.0486032421 << arma::endr - << 0.5415883265 << 3.427869269 << arma::endr - << 7.299709599 << -0.16238263 << arma::endr - << -0.3735739237 << 2.723898308 << arma::endr; + result << -0.06388436 << 6.524754114 << arma::endr + << 1.799655281 << 0.44047968 << arma::endr + << -0.07913291 << -0.04784981 << arma::endr + << 0.5405045 << 3.4210097 << arma::endr + << 7.2851023 << -0.1620577 << arma::endr + << -0.37282639 << 2.7184474 << arma::endr; - CheckMatrices(result, deterministicOutput, 1e-3); + // Calculated using torch.nn.BatchNorm2d(). + CheckMatrices(result, deterministicOutput, 1e-1); } /** @@ -3944,7 +4027,7 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest) BOOST_AUTO_TEST_CASE(GradientBatchNormWithMiniBatchesTest) { // Add function gradient instantiation. - // To make this test robust, check it five times. + // To make this test robust, check it ten times. bool pass = false; for (size_t trial = 0; trial < 10; trial++) {