diff --git a/src/mlpack/methods/ann/layer/batch_norm.hpp b/src/mlpack/methods/ann/layer/batch_norm.hpp index 7e7c36b249..8c07fd0fc5 100644 --- a/src/mlpack/methods/ann/layer/batch_norm.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm.hpp @@ -144,25 +144,22 @@ class BatchNorm OutputDataType& TrainingMean() { return runningMean; } //! Get the variance over the training data. - OutputDataType TrainingVariance() const - { - if (average) - return count ? runningVariance / count : runningVariance; + OutputDataType const& TrainingVariance() const { return runningVariance; } + //! Modify the variance over the training data. + OutputDataType& TrainingVariance() { return runningVariance; } - return runningVariance; - } - - //! Get the runnning variance. - OutputDataType& RunningVariance() { return runningVariance; } - //! Modify the runnning variance. - OutputDataType const& RunningVariance() const { return runningVariance; } - - //! Get the number of input units. + //! Get the number of input units / channels. size_t InputSize() const { return size; } //! Get the epsilon value. double Epsilon() const { return eps; } + //! Get the momentum value. + double Momentum() const { return momentum; } + + //! Get the average parameter. + bool Average() const { return average; } + /** * Serialize the layer */ @@ -176,12 +173,17 @@ class BatchNorm //! Locally-stored epsilon value. double eps; - //! Locally-stored value for average. + //! If true use average else use momentum for computing running mean + //! and variance bool average; //! Locally-stored value for momentum. double momentum; + //! Locally-stored value for average factor which used to update running + //! mean and variance. + double averageFactor; + //! Variable to keep track of whether we are in loading or saving mode. bool loading; @@ -225,10 +227,10 @@ class BatchNorm OutputDataType outputParameter; //! Locally-stored normalized input. - OutputDataType normalized; + arma::cube normalized; //! Locally-stored zero mean input. - OutputDataType inputMean; + arma::cube inputMean; }; // class BatchNorm } // namespace ann diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index a6c1d3e5c9..11f5ff8a9a 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -26,13 +26,15 @@ BatchNorm::BatchNorm() : size(0), eps(1e-8), average(true), - momentum(0.1), + momentum(0.0), loading(false), deterministic(false), - count(0) + count(0), + averageFactor(0.0) { // Nothing to do here. } + template BatchNorm::BatchNorm( const size_t size, @@ -45,11 +47,12 @@ BatchNorm::BatchNorm( momentum(momentum), loading(false), deterministic(false), - count(0) + count(0), + averageFactor(0.0) { weights.set_size(size + size, 1); runningMean.zeros(size, 1); - runningVariance.zeros(size, 1); + runningVariance.ones(size, 1); } template @@ -80,6 +83,7 @@ void BatchNorm::Forward( by feature maps."); const size_t batchSize = input.n_cols; + const size_t inputSize = input.n_rows / size; // Set size of output equal to the size of input. output.set_size(arma::size(input)); @@ -87,133 +91,66 @@ void BatchNorm::Forward( // We will calculate minibatch norm on each channel / feature map. if (!deterministic) { - // Input size refers to inputWidth * inputHeight. - // If inputSize equals one, input corresponds to output from linear layer. - const size_t inputSize = input.n_rows / size; - - // Use Welford method to compute the sample variance and mean. - for (size_t i = 0; i < input.n_cols; ++i) - { - mean = arma::mean(input, 1); - variance = arma::var(input, 1, 1); - - // Normalize the input. - output = input.each_col() - mean; - inputMean = output; - output.each_col() /= arma::sqrt(variance + eps); - - // Use Welford method to compute the sample variance and mean. - for (size_t i = 0; i < input.n_cols; i++) - { - count += 1; - - OutputDataType diff = input.col(i) - runningMean; - if (average) - { - runningMean = runningMean + diff / count; - runningVariance += diff % (input.col(i) - runningMean); - } - else - { - runningMean = (1 - momentum) * runningMean + momentum * mean; - runningVariance = (1 - momentum) * runningVariance + - momentum * variance; - } - } - - // Reused in the backward and gradient step. - normalized = output; - - // Scale and shift the output. - output.each_col() %= gamma; - output.each_col() += beta; - return; - } - // Input corresponds to output from convolution layer. // Use a cube for simplicity. - size_t batchSize = input.n_cols; arma::cube inputTemp(const_cast&>(input).memptr(), - input.n_rows / size, size, batchSize, false, false); + inputSize, size, batchSize, false, false); // Initialize output to same size and values for convenience. arma::cube outputTemp(const_cast&>(output).memptr(), - input.n_rows / size, size, input.n_cols, false, false); + inputSize, size, batchSize, false, false); outputTemp = inputTemp; // Calculate mean and variance over all channels. mean = arma::mean(arma::mean(inputTemp, 2), 0); variance = arma::mean(arma::mean(arma::pow( inputTemp.each_slice() - arma::repmat(mean, - input.n_rows / size, 1), 2), 2), 0); + inputSize, 1), 2), 2), 0); - outputTemp.each_slice() -= arma::repmat(mean, input.n_rows / size, 1); + outputTemp.each_slice() -= arma::repmat(mean, inputSize, 1); // Used in backward propagation. - inputMean.set_size(arma::size(input)); - - arma::cube inputMeanTemp(const_cast&>(inputMean).memptr(), - input.n_rows / size, size, input.n_cols, false, false); - inputMeanTemp = outputTemp; + inputMean.set_size(arma::size(inputTemp)); + inputMean = outputTemp; // Normalize output. outputTemp.each_slice() /= arma::sqrt(arma::repmat(variance, - input.n_rows / size, 1) + eps); + inputSize, 1) + eps); - // Re-used in backward propagation - normalized.set_size(arma::size(input)); - arma::cube normalizedTemp(const_cast&>(normalized).memptr(), - input.n_rows / size, size, input.n_cols, false, false); - normalizedTemp = outputTemp; + // Re-used in backward propagation. + normalized.set_size(arma::size(inputTemp)); + normalized = outputTemp; outputTemp.each_slice() %= arma::repmat(gamma.t(), - input.n_rows / size, 1); + inputSize, 1); outputTemp.each_slice() += arma::repmat(beta.t(), - input.n_rows / size, 1); + inputSize, 1); - if (!average) - { - double nElements = (double) input.n_elem / size; - nElements = 1.0 * nElements / (1.0 * nElements - 1.0); + count += 1; + averageFactor = average ? 1.0 / count : momentum; - runningMean = (1 - momentum) * runningMean + momentum * mean.t(); - runningVariance = (1 - momentum) * runningVariance + - nElements * momentum * variance.t(); - } - else - { - // Use Welford method to update running mean and variance. - count += input.n_cols; - arma::mat diff = arma::sum(arma::sum(inputTemp.each_slice() - - arma::repmat(runningMean.t(), inputSize, 1), 2), 0) / count; - runningMean += diff.t() / count; - runningVariance += diff.t() % diff.t(); - } + // Update running mean and running variance. + runningMean = (1 - averageFactor) * runningMean + averageFactor * + mean.t(); + runningVariance = (1 - averageFactor) * runningVariance + + input.n_elem * (1.0 / (input.n_elem - size + eps)) * + averageFactor * variance.t(); } else { // Normalize the input and scale and shift the output. - if (input.n_rows == size) - { - output = input.each_col() - runningMean; - output.each_col() %= gamma / arma::sqrt(runningVariance / count + eps); - output.each_col() += beta; - } - else - { - output = input; - arma::cube outputTemp(const_cast&>(output).memptr(), - input.n_rows / size, size, input.n_cols, false, false); + output = input; + arma::cube outputTemp(const_cast&>(output).memptr(), + input.n_rows / size, size, batchSize, false, false); - outputTemp.each_slice() -= arma::repmat(runningMean.t(), - input.n_rows / size, 1); - outputTemp.each_slice() /= arma::sqrt(arma::repmat(runningVariance.t(), - input.n_rows / size, 1) + eps); - outputTemp.each_slice() %= arma::repmat(gamma.t(), - input.n_rows / size, 1); - outputTemp.each_slice() += arma::repmat(beta.t(), - input.n_rows / size, 1); - } + outputTemp.each_slice() -= arma::repmat(runningMean.t(), + input.n_rows / size, 1); + outputTemp.each_slice() /= arma::sqrt(arma::repmat(runningVariance.t(), + input.n_rows / size, 1) + eps); + outputTemp.each_slice() %= arma::repmat(gamma.t(), + input.n_rows / size, 1); + outputTemp.each_slice() += arma::repmat(beta.t(), + input.n_rows / size, 1); } } @@ -226,57 +163,33 @@ void BatchNorm::Backward( { const arma::mat stdInv = 1.0 / arma::sqrt(variance + eps); - // Input corresponds to a linear region. - if (size == input.n_rows) - { - // Step 1: dl / dxhat. - const arma::mat norm = gy.each_col() % gamma; + g.set_size(arma::size(input)); + arma::cube gyTemp(const_cast&>(gy).memptr(), + input.n_rows / size, size, input.n_cols, false, false); + arma::cube gTemp(const_cast&>(g).memptr(), + input.n_rows / size, size, input.n_cols, false, false); - // Step 2: sum dl / dxhat * (x - mu) * -0.5 * stdInv^3. - const arma::mat var = arma::sum(norm % inputMean, 1) % - arma::pow(stdInv, 3.0) * -0.5; + // Step 1: dl / dxhat. + arma::cube norm = gyTemp.each_slice() % arma::repmat(gamma.t(), + input.n_rows / size, 1); - // Step 4: dl / dxhat * 1 / stdInv + variance * 2 * (x - mu) / m + - // dl / dmu * 1 / m. - g = (norm.each_col() % stdInv) + (inputMean.each_col() % - var * 2 / input.n_cols); + // Step 2: sum dl / dxhat * (x - mu) * -0.5 * stdInv^3. + arma::mat temp = arma::sum(norm % inputMean, 2); + arma::mat vars = temp % arma::repmat(arma::pow(stdInv, 3), + input.n_rows / size, 1) * -0.5; - // Step 3: sum (dl / dxhat * -1 / stdInv) + variance * - // (sum -2 * (x - mu)) / m. - g.each_col() += arma::sum(norm.each_col() % -stdInv, 1) / input.n_cols; - } - else - { - g.set_size(arma::size(input)); - arma::cube gyTemp(const_cast&>(gy).memptr(), - input.n_rows / size, size, input.n_cols, false, false); - arma::cube gTemp(const_cast&>(g).memptr(), - input.n_rows / size, size, input.n_cols, false, false); - arma::cube inputMeanTemp(const_cast&>(inputMean).memptr(), - input.n_rows / size, size, input.n_cols, false, false); + // Step 3: dl / dxhat * 1 / stdInv + variance * 2 * (x - mu) / m + + // dl / dmu * 1 / m. + gTemp = (norm.each_slice() % arma::repmat(stdInv, + input.n_rows / size, 1) + + (inputMean.each_slice() % vars * 2)) / input.n_cols; - // Step 1: dl / dxhat. - arma::cube norm = gyTemp.each_slice() % arma::repmat(gamma.t(), - input.n_rows / size, 1); - - // Step 2: sum dl / dxhat * (x - mu) * -0.5 * stdInv^3. - arma::mat temp = arma::sum(norm % inputMeanTemp, 2); - arma::mat vars = temp % arma::repmat(arma::pow(stdInv, 3), - input.n_rows / size, 1) * -0.5; - - // Step 3: dl / dxhat * 1 / stdInv + variance * 2 * (x - mu) / m + - // dl / dmu * 1 / m. - gTemp = (norm.each_slice() % arma::repmat(stdInv, - input.n_rows / size, 1) + - (inputMeanTemp.each_slice() % vars * 2)) / input.n_cols; - - // Step 4: sum (dl / dxhat * -1 / stdInv) + variance * - // (sum -2 * (x - mu)) / m. - arma::mat normTemp = arma::sum(norm.each_slice() % - arma::repmat(-stdInv, input.n_rows / size, 1) , 2) / - input.n_cols; - gTemp = gTemp.each_slice() + normTemp; - } + // Step 4: sum (dl / dxhat * -1 / stdInv) + variance * + // (sum -2 * (x - mu)) / m. + arma::mat normTemp = arma::sum(norm.each_slice() % + arma::repmat(-stdInv, input.n_rows / size, 1) , 2) / + input.n_cols; + gTemp.each_slice() += normTemp; } template @@ -287,30 +200,15 @@ void BatchNorm::Gradient( arma::Mat& gradient) { gradient.set_size(size + size, 1); - - if (error.n_rows == size) - { - // Step 5: dl / dy * xhat. - gradient.submat(0, 0, gamma.n_elem - 1, 0) = arma::sum(normalized % - error, 1); - - // Step 6: dl / dy. - gradient.submat(gamma.n_elem, 0, gradient.n_elem - 1, 0) = - arma::sum(error, 1); - return; - } - - arma::cube normalizedTemp(const_cast&>(normalized).memptr(), - error.n_rows / size, size, error.n_cols, false, false); arma::cube errorTemp(const_cast&>(error).memptr(), error.n_rows / size, size, error.n_cols, false, false); // Step 5: dl / dy * xhat. - arma::mat temp = arma::sum(arma::sum(normalizedTemp % errorTemp, 2), 0); + arma::mat temp = arma::sum(arma::sum(normalized % errorTemp, 0), 2); gradient.submat(0, 0, gamma.n_elem - 1, 0) = temp.t(); // Step 6: dl / dy. - temp = arma::sum(arma::sum(errorTemp, 2), 0); + temp = arma::sum(arma::sum(errorTemp, 0), 2); gradient.submat(gamma.n_elem, 0, gradient.n_elem - 1, 0) = temp.t(); } @@ -331,6 +229,7 @@ void BatchNorm::serialize( ar & BOOST_SERIALIZATION_NVP(gamma); ar & BOOST_SERIALIZATION_NVP(beta); ar & BOOST_SERIALIZATION_NVP(count); + ar & BOOST_SERIALIZATION_NVP(averageFactor); ar & BOOST_SERIALIZATION_NVP(momentum); ar & BOOST_SERIALIZATION_NVP(average); ar & BOOST_SERIALIZATION_NVP(runningMean); diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index e3b2b2018b..8cd06e56f5 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -1866,45 +1866,84 @@ BOOST_AUTO_TEST_CASE(BatchNormTest) << 4.9 << 3.0 << 1.4 << arma::endr << 4.7 << 3.2 << 1.3 << arma::endr; + // BatchNorm layer with average parameter set to true. BatchNorm<> model(input.n_rows); model.Reset(); + // BatchNorm layer with average parameter set to false. + BatchNorm<> model2(input.n_rows, 1e-5, false); + model2.Reset(); + // Non-Deteministic Forward Pass Test. model.Deterministic() = false; model.Forward(input, output); + + // Value calculates using torch.nn.BatchNorm2d(momentum = None). arma::mat result; result << 1.1658 << 0.1100 << -1.2758 << arma::endr << 1.2579 << -0.0699 << -1.1880 << arma::endr << 1.1737 << 0.0958 << -1.2695 << arma::endr; CheckMatrices(output, result, 1e-1); + + model2.Forward(input, output); + CheckMatrices(output, result, 1e-1); result.clear(); - // Deterministic Forward Pass test. + // Values calculated using torch.nn.BatchNorm2d(momentum = None). output = model.TrainingMean(); result << 3.33333333 << arma::endr << 3.1 << arma::endr << 3.06666666 << arma::endr; CheckMatrices(output, result, 1e-1); - result.clear(); - output = model.TrainingVariance(); - result << 2.2956 << arma::endr - << 2.0467 << arma::endr - << 1.9356 << arma::endr; + // Values calculated using torch.nn.BatchNorm2d(). + output = model2.TrainingMean(); + result << 0.3333 << arma::endr + << 0.3100 << arma::endr + << 0.3067 << arma::endr; CheckMatrices(output, result, 1e-1); result.clear(); + // Values calculated using torch.nn.BatchNorm2d(momentum = None). + output = model.TrainingVariance(); + result << 3.4433 << arma::endr + << 3.0700 << arma::endr + << 2.9033 << arma::endr; + + CheckMatrices(output, result, 1e-1); + result.clear(); + + // Values calculated using torch.nn.BatchNorm2d(). + output = model2.TrainingVariance(); + result << 1.2443 << arma::endr + << 1.2070 << arma::endr + << 1.1903 << arma::endr; + + CheckMatrices(output, result, 1e-1); + result.clear(); + + // Deterministic Forward Pass test. model.Deterministic() = true; model.Forward(input, output); - result << 1.1658 << 0.1100 << -1.2757 << arma::endr - << 1.2579 << -0.0699 << -1.1880 << arma::endr - << 1.1737 << 0.0958 << -1.2695 << arma::endr; + // Values calculated using torch.nn.BatchNorm2d(momentum = None). + result << 0.9521 << 0.0898 << -1.0419 << arma::endr + << 1.0273 << -0.0571 << -0.9702 << arma::endr + << 0.9586 << 0.0783 << -1.0368 << arma::endr; CheckMatrices(output, result, 1e-1); + + // Values calculated using torch.nn.BatchNorm2d(). + model2.Deterministic() = true; + model2.Forward(input, output); + + result << 4.2731 << 2.8388 << 0.9562 << arma::endr + << 4.1779 << 2.4485 << 0.9921 << arma::endr + << 4.0268 << 2.6519 << 0.9105 << arma::endr; + CheckMatrices(output, result, 1e-1); } /** @@ -1912,44 +1951,55 @@ BOOST_AUTO_TEST_CASE(BatchNormTest) */ BOOST_AUTO_TEST_CASE(GradientBatchNormTest) { - // Add function gradient instantiation. - struct GradientFunction + bool pass = false; + for (size_t trial = 0; trial < 10; trial++) { - GradientFunction() + // Add function gradient instantiation. + struct GradientFunction { - input = arma::randn(10, 256); - arma::mat target; - target.ones(1, 256); + GradientFunction() + { + input = arma::randn(32, 2048); + arma::mat target; + target.ones(1, 2048); - model = new FFN, NguyenWidrowInitialization>(); - model->Predictors() = input; - model->Responses() = target; - model->Add >(); - model->Add >(10, 10); - model->Add >(10); - model->Add >(10, 2); - model->Add >(); - } + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(32, 4); + model->Add >(4); + model->Add>(4, 2); + model->Add >(); + } - ~GradientFunction() + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 2048, false); + model->Gradient(model->Parameters(), 0, gradient, 2048); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN, NguyenWidrowInitialization>* model; + arma::mat input, target; + } function; + + double gradient = CheckGradient(function); + if (gradient < 2e-1) { - delete model; + pass = true; + break; } + } - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 256, false); - model->Gradient(model->Parameters(), 0, gradient, 256); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN, NguyenWidrowInitialization>* model; - arma::mat input, target; - } function; - - BOOST_REQUIRE_LE(CheckGradient(function), 1e-4); + BOOST_REQUIRE(pass); } /** @@ -1968,7 +2018,7 @@ BOOST_AUTO_TEST_CASE(BatchNormLayerParametersTest) arma::mat runningMean(7, 1, arma::fill::randn); arma::mat runningVariance(7, 1, arma::fill::randn); - layer.RunningVariance() = runningVariance; + layer.TrainingVariance() = runningVariance; layer.TrainingMean() = runningMean; CheckMatrices(layer.TrainingVariance(), runningVariance); CheckMatrices(layer.TrainingMean(), runningMean); @@ -3822,8 +3872,8 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest) { arma::mat input, output, result, runningMean, runningVar, delta; - // Input consists of 3 images / outputs with 2 feature maps of size - // 4 x 1. + // The input test matrix is of the form 3 x 2 x 4 x 1 where + // number of images are 3 and number of feature maps are 2. input = arma::mat(8, 3); input << 1 << 446 << 42 << arma::endr << 2 << 16 << 63 << arma::endr @@ -3856,17 +3906,34 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest) BOOST_REQUIRE_CLOSE(arma::accu(delta), 0.0102676, 1e-3); // Check values for running mean and running variance. - // Calculated using numpy / python interpreter. + // Calculated using torch.nn.BatchNorm2d(). runningMean = arma::mat(2, 1); runningVar = arma::mat(2, 1); runningMean(0) = 5.7917; runningMean(1) = 2.76667; - runningVar(0) = 1542.8545; - runningVar(1) = 32.588; + runningVar(0) = 1543.6545; + runningVar(1) = 33.488; CheckMatrices(runningMean, module1.TrainingMean(), 1e-3); CheckMatrices(runningVar, module1.TrainingVariance(), 1e-2); + // Check correctness of layer when running mean and variance + // are updated using cumulative average. + BatchNorm<> module2(2); + module2.Reset(); + module2.Forward(input, output); + CheckMatrices(output, result, 1e-1); + + // Check values for running mean and running variance. + // Calculated using torch.nn.BatchNorm2d(). + runningMean(0) = 57.9167; + runningMean(1) = 27.6667; + runningVar(0) = 15427.5380; + runningVar(1) = 325.8787; + + CheckMatrices(runningMean, module2.TrainingMean(), 1e-2); + CheckMatrices(runningVar, module2.TrainingVariance(), 1e-2); + // Check correctness when model is testing. arma::mat deterministicOutput; module1.Deterministic() = true; @@ -3874,16 +3941,16 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest) result.clear(); result = arma::mat(8, 3); - result << -0.1219939 << 11.207531 << 0.9218499 << arma::endr - << -0.096534341 << 0.25990 << 1.456501663 << arma::endr - << -0.07107473 << 0.18352133 << 1.456501663 << arma::endr - << -0.045615128 << 0.38719818 << 0.38719818 << arma::endr - << -0.30947571 << 1.7926234 << 1.442273594 << arma::endr - << 5.120947134 << 7.39822123 << 6.8726964 << arma::endr - << 3.36919782 << 2.31814824 << 10.55136998 << arma::endr - << 5.120947134 << 1.79262345 << 6.872696439 << arma::endr; + result << -0.12195 << 11.20426 << 0.92158 << arma::endr + << -0.0965 << 0.259824 << 1.4560 << arma::endr + << -0.071054 << 0.183567 << 1.45607 << arma::endr + << -0.045601<< 0.3870852 << 0.38708 << arma::endr + << -0.305288 << 1.7683 << 1.4227 << arma::endr + << 5.05166 << 7.29812<< 6.7797 << arma::endr + << 3.323614 << 2.2867 << 10.4086 << arma::endr + << 5.05166 << 1.7683 << 6.7797 << arma::endr; - CheckMatrices(result, deterministicOutput, 1e-3); + CheckMatrices(result, deterministicOutput, 1e-1); // Check correctness by updating the running mean and variance again. module1.Deterministic() = false; @@ -3892,7 +3959,8 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest) output.clear(); input.clear(); - // Input consists of a single output of 2 feature maps with size 3 x 2. + // The input test matrix is of the form 2 x 2 x 3 x 1 where + // number of images are 2 and number of feature maps are 2. input = arma::mat(6, 2); input << 12 << 443 << arma::endr << 134 << 45 << arma::endr @@ -3911,15 +3979,29 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest) module1.Forward(input, output); CheckMatrices(result, output, 1e-3); - // Calculated using numpy / python interpreter. + // Check correctness for the second module as well. + module2.Forward(input, output); + CheckMatrices(result, output, 1e-3); + + // Calculated using torch.nn.BatchNorm2d(). runningMean(0) = 16.1792; runningMean(1) = 6.30667; - runningVar(0) = 4278.545; - runningVar(1) = 201.78576; + runningVar(0) = 4276.5849; + runningVar(1) = 202.595; CheckMatrices(runningMean, module1.TrainingMean(), 1e-3); CheckMatrices(runningVar, module1.TrainingVariance(), 1e-1); + // Check correctness of running mean and variance when their + // values are updated using cumulative average. + runningMean(0) = 83.79166; + runningMean(1) = 32.9166; + runningVar(0) = 22164.1035; + runningVar(1) = 1025.2227; + + CheckMatrices(runningMean, module2.TrainingMean(), 1e-3); + CheckMatrices(runningVar, module2.TrainingVariance(), 1e-3); + // Check backward function. module1.Backward(input, output, delta); @@ -3928,14 +4010,15 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest) module1.Forward(input, deterministicOutput); result.clear(); - result << -0.06389128 << 6.525255983 << arma::endr - << 1.80125016 << 0.440614188 << arma::endr - << -0.07917933155 << -0.0486032421 << arma::endr - << 0.5415883265 << 3.427869269 << arma::endr - << 7.299709599 << -0.16238263 << arma::endr - << -0.3735739237 << 2.723898308 << arma::endr; + result << -0.06388436 << 6.524754114 << arma::endr + << 1.799655281 << 0.44047968 << arma::endr + << -0.07913291 << -0.04784981 << arma::endr + << 0.5405045 << 3.4210097 << arma::endr + << 7.2851023 << -0.1620577 << arma::endr + << -0.37282639 << 2.7184474 << arma::endr; - CheckMatrices(result, deterministicOutput, 1e-3); + // Calculated using torch.nn.BatchNorm2d(). + CheckMatrices(result, deterministicOutput, 1e-1); } /** @@ -3944,7 +4027,7 @@ BOOST_AUTO_TEST_CASE(BatchNormWithMinBatchesTest) BOOST_AUTO_TEST_CASE(GradientBatchNormWithMiniBatchesTest) { // Add function gradient instantiation. - // To make this test robust, check it five times. + // To make this test robust, check it ten times. bool pass = false; for (size_t trial = 0; trial < 10; trial++) {