From b1f16ce5d8edab206ddef61e50aae55ca61f45b8 Mon Sep 17 00:00:00 2001 From: Omar Shrit Date: Mon, 18 Sep 2023 19:55:59 +0200 Subject: [PATCH 1/4] Keep the batch normalization aside for a bit Signed-off-by: Omar Shrit --- src/mlpack/methods/ann/ffn_impl.hpp | 2 +- src/mlpack/methods/ann/layer/c_relu_impl.hpp | 5 +++-- src/mlpack/methods/ann/layer/convolution.hpp | 4 ++-- src/mlpack/methods/ann/layer/grouped_convolution.hpp | 4 ++-- src/mlpack/methods/ann/layer/grouped_convolution_impl.hpp | 4 ++-- src/mlpack/methods/ann/layer/leaky_relu.hpp | 8 ++++---- src/mlpack/methods/ann/layer/leaky_relu_impl.hpp | 2 +- src/mlpack/methods/ann/layer/serialization.hpp | 1 + 8 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/mlpack/methods/ann/ffn_impl.hpp b/src/mlpack/methods/ann/ffn_impl.hpp index 493b2f9867..a31fdddfac 100644 --- a/src/mlpack/methods/ann/ffn_impl.hpp +++ b/src/mlpack/methods/ann/ffn_impl.hpp @@ -468,7 +468,7 @@ typename MatType::elem_type FFN< res += EvaluateWithGradient(parameters, 0, gradient, 1); for (size_t i = 1; i < predictors.n_cols; ++i) { - arma::mat tmpGradient(gradient.n_rows, gradient.n_cols); + MatType tmpGradient(gradient.n_rows, gradient.n_cols); res += EvaluateWithGradient(parameters, i, tmpGradient, 1); gradient += tmpGradient; } diff --git a/src/mlpack/methods/ann/layer/c_relu_impl.hpp b/src/mlpack/methods/ann/layer/c_relu_impl.hpp index 3a99d9e75e..c58972d9ef 100644 --- a/src/mlpack/methods/ann/layer/c_relu_impl.hpp +++ b/src/mlpack/methods/ann/layer/c_relu_impl.hpp @@ -68,13 +68,14 @@ template void CReLUType::Forward( const MatType& input, MatType& output) { + typename MatType::elem_type zero = 0.0; #pragma omp for for (size_t i = 0; i < (size_t) input.n_cols; ++i) { for (size_t j = 0; j < (size_t) input.n_rows; ++j) { - output(j, i) = std::max(input(j, i), 0.0); - output(j + input.n_rows, i) = std::max(-input(j, i), 0.0); + output(j, i) = std::max(input(j, i), zero); + output(j + input.n_rows, i) = std::max(-input(j, i), zero); } } } diff --git a/src/mlpack/methods/ann/layer/convolution.hpp b/src/mlpack/methods/ann/layer/convolution.hpp index 582fd9be78..865ee52ce0 100644 --- a/src/mlpack/methods/ann/layer/convolution.hpp +++ b/src/mlpack/methods/ann/layer/convolution.hpp @@ -374,10 +374,10 @@ class ConvolutionType : public Layer arma::Cube gradientTemp; //! Locally-stored padding layer. - Padding padding; + PaddingType padding; //! Locally-stored padding layer for backward pass. - Padding paddingBackward; + PaddingType paddingBackward; //! Type of padding. std::string paddingType; diff --git a/src/mlpack/methods/ann/layer/grouped_convolution.hpp b/src/mlpack/methods/ann/layer/grouped_convolution.hpp index f1b1af4383..072d98589f 100644 --- a/src/mlpack/methods/ann/layer/grouped_convolution.hpp +++ b/src/mlpack/methods/ann/layer/grouped_convolution.hpp @@ -390,10 +390,10 @@ class GroupedConvolutionType : public Layer arma::Cube gradientTemp; //! Locally-stored padding layer. - Padding padding; + PaddingType padding; //! Locally-stored padding layer for backward pass. - Padding paddingBackward; + PaddingType paddingBackward; //! Type of padding. std::string paddingType; diff --git a/src/mlpack/methods/ann/layer/grouped_convolution_impl.hpp b/src/mlpack/methods/ann/layer/grouped_convolution_impl.hpp index 5c946014bb..33f5b00006 100644 --- a/src/mlpack/methods/ann/layer/grouped_convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/grouped_convolution_impl.hpp @@ -612,7 +612,7 @@ void GroupedConvolutionType< InitializeSamePadding(); } - padding = Padding(padWLeft, padWRight, padHTop, padHBottom); + padding = PaddingType(padWLeft, padWRight, padHTop, padHBottom); padding.InputDimensions() = this->inputDimensions; padding.ComputeOutputDimensions(); @@ -650,7 +650,7 @@ void GroupedConvolutionType< apparentHeight = (this->outputDimensions[1] - 1) * strideHeight + kernelHeight; - paddingBackward = Padding(0, padding.OutputDimensions()[0] - + paddingBackward = PaddingType(0, padding.OutputDimensions()[0] - apparentWidth, 0, padding.OutputDimensions()[1] - apparentHeight); paddingBackward.InputDimensions() = std::vector({ apparentWidth, apparentHeight, inMaps * higherInDimensions }); diff --git a/src/mlpack/methods/ann/layer/leaky_relu.hpp b/src/mlpack/methods/ann/layer/leaky_relu.hpp index 3571a60353..8d4eb11522 100644 --- a/src/mlpack/methods/ann/layer/leaky_relu.hpp +++ b/src/mlpack/methods/ann/layer/leaky_relu.hpp @@ -47,7 +47,7 @@ class LeakyReLUType : public Layer * * @param alpha Non zero gradient. */ - LeakyReLUType(const double alpha = 0.03); + LeakyReLUType(const typename MatType::elem_type alpha = 0.03); //! Clone the LeakyReLUType object. This handles polymorphism correctly. LeakyReLUType* Clone() const { return new LeakyReLUType(*this); } @@ -85,9 +85,9 @@ class LeakyReLUType : public Layer void Backward(const MatType& input, const MatType& gy, MatType& g); //! Get the non zero gradient. - double const& Alpha() const { return alpha; } + typename MatType::elem_type const& Alpha() const { return alpha; } //! Modify the non zero gradient. - double& Alpha() { return alpha; } + typename MatType::elem_type& Alpha() { return alpha; } //! Serialize the layer. template @@ -95,7 +95,7 @@ class LeakyReLUType : public Layer private: //! Leakyness Parameter in the range 0 -LeakyReLUType::LeakyReLUType(const double alpha) : +LeakyReLUType::LeakyReLUType(const typename MatType::elem_type alpha) : Layer(), alpha(alpha) { diff --git a/src/mlpack/methods/ann/layer/serialization.hpp b/src/mlpack/methods/ann/layer/serialization.hpp index a32c362c0c..508ddbdaf0 100644 --- a/src/mlpack/methods/ann/layer/serialization.hpp +++ b/src/mlpack/methods/ann/layer/serialization.hpp @@ -76,5 +76,6 @@ CEREAL_REGISTER_TYPE(mlpack::FTSwishType<__VA_ARGS__>); \ CEREAL_REGISTER_MLPACK_LAYERS(arma::mat); +CEREAL_REGISTER_MLPACK_LAYERS(arma::fmat); #endif From af38b13affb9c378278dd610a19b589aabe3c1aa Mon Sep 17 00:00:00 2001 From: Omar Shrit Date: Mon, 18 Sep 2023 20:16:19 +0200 Subject: [PATCH 2/4] Compiling Signed-off-by: Omar Shrit --- .../methods/ann/layer/batch_norm_impl.hpp | 32 +++++++++---------- .../methods/ann/layer/convolution_impl.hpp | 4 +-- .../methods/ann/layer/mean_pooling_impl.hpp | 2 +- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index 213a5c217a..336c5c06fa 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -216,26 +216,26 @@ void BatchNormType::Forward( // Calculate mean and variance over all channels. MatType mean = arma::sum(arma::sum(inputTemp, 2), 0) / m; variance = arma::sum(arma::sum(arma::pow( - inputTemp.each_slice() - arma::repmat(mean, + inputTemp.each_slice() - arma::repmat(mean, inputSize, 1), 2), 2), 0) / m; - outputTemp.each_slice() -= arma::repmat(mean, inputSize, 1); + outputTemp.each_slice() -= arma::repmat(mean, inputSize, 1); // Used in backward propagation. inputMean.set_size(arma::size(inputTemp)); inputMean = outputTemp; // Normalize output. - outputTemp.each_slice() /= arma::sqrt(arma::repmat(variance, + outputTemp.each_slice() /= arma::sqrt(arma::repmat(variance, inputSize, 1) + eps); // Re-used in backward propagation. normalized.set_size(arma::size(inputTemp)); normalized = outputTemp; - outputTemp.each_slice() %= arma::repmat(gamma.t(), + outputTemp.each_slice() %= arma::repmat(gamma.t(), inputSize, 1); - outputTemp.each_slice() += arma::repmat(beta.t(), + outputTemp.each_slice() += arma::repmat(beta.t(), inputSize, 1); count += 1; @@ -260,13 +260,13 @@ void BatchNormType::Forward( const_cast(output).memptr(), inputSize, size, batchSize * higherDimension, false, false); - outputTemp.each_slice() -= arma::repmat(runningMean.t(), + outputTemp.each_slice() -= arma::repmat(runningMean.t(), inputSize, 1); - outputTemp.each_slice() /= arma::sqrt(arma::repmat(runningVariance.t(), + outputTemp.each_slice() /= arma::sqrt(arma::repmat(runningVariance.t(), inputSize, 1) + eps); - outputTemp.each_slice() %= arma::repmat(gamma.t(), + outputTemp.each_slice() %= arma::repmat(gamma.t(), inputSize, 1); - outputTemp.each_slice() += arma::repmat(beta.t(), + outputTemp.each_slice() += arma::repmat(beta.t(), inputSize, 1); } } @@ -277,7 +277,7 @@ void BatchNormType::Backward( const MatType& gy, MatType& g) { - const arma::mat stdInv = 1.0 / arma::sqrt(variance + eps); + const MatType stdInv = 1.0 / arma::sqrt(variance + eps); const size_t batchSize = input.n_cols; const size_t inputSize = inputDimension; @@ -292,7 +292,7 @@ void BatchNormType::Backward( // Step 1: dl / dxhat. arma::Cube norm = - gyTemp.each_slice() % arma::repmat(gamma.t(), inputSize, 1); + gyTemp.each_slice() % arma::repmat(gamma.t(), inputSize, 1); // Step 2: sum dl / dxhat * (x - mu) * -0.5 * stdInv^3. MatType temp = arma::sum(arma::sum(norm % inputMean, 2), 0); @@ -300,17 +300,17 @@ void BatchNormType::Backward( // Step 3: dl / dxhat * 1 / stdInv + variance * 2 * (x - mu) / m + // dl / dmu * 1 / m. - gTemp = (norm.each_slice() % arma::repmat(stdInv, + gTemp = (norm.each_slice() % arma::repmat(stdInv, inputSize, 1)) + - ((inputMean.each_slice() % arma::repmat(vars, inputSize, 1) * 2.0) / m); + ((inputMean.each_slice() % arma::repmat(vars, inputSize, 1) * 2.0) / m); // Step 4: sum (dl / dxhat * -1 / stdInv) + variance * // sum (-2 * (x - mu)) / m. MatType normTemp = arma::sum(arma::sum((norm.each_slice() % - arma::repmat(-stdInv, inputSize, 1)) + - (inputMean.each_slice() % arma::repmat(vars, inputSize, 1) * (-2.0) / m), + arma::repmat(-stdInv, inputSize, 1)) + + (inputMean.each_slice() % arma::repmat(vars, inputSize, 1) * (-2.0) / m), 2), 0) / m; - gTemp.each_slice() += arma::repmat(normTemp, inputSize, 1); + gTemp.each_slice() += arma::repmat(normTemp, inputSize, 1); } template diff --git a/src/mlpack/methods/ann/layer/convolution_impl.hpp b/src/mlpack/methods/ann/layer/convolution_impl.hpp index 567cc85f85..3b83e05566 100644 --- a/src/mlpack/methods/ann/layer/convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/convolution_impl.hpp @@ -573,7 +573,7 @@ void ConvolutionType< InitializeSamePadding(); } - padding = Padding(padWLeft, padWRight, padHTop, padHBottom); + padding = PaddingType(padWLeft, padWRight, padHTop, padHBottom); padding.InputDimensions() = this->inputDimensions; padding.ComputeOutputDimensions(); @@ -600,7 +600,7 @@ void ConvolutionType< apparentHeight = (this->outputDimensions[1] - 1) * strideHeight + kernelHeight; - paddingBackward = Padding(0, padding.OutputDimensions()[0] - + paddingBackward = PaddingType(0, padding.OutputDimensions()[0] - apparentWidth, 0, padding.OutputDimensions()[1] - apparentHeight); paddingBackward.InputDimensions() = std::vector({ apparentWidth, apparentHeight, inMaps * higherInDimensions }); diff --git a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp index 420e7c12ce..5f1cf434d0 100644 --- a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp +++ b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp @@ -376,7 +376,7 @@ void MeanPoolingType::Unpooling( rowEnd = output.n_rows - 1; } - arma::mat OutputArea = output(arma::span(i, rowEnd), arma::span(j, colEnd)); + MatType OutputArea = output(arma::span(i, rowEnd), arma::span(j, colEnd)); unpooledError = arma::Mat(OutputArea.n_rows, OutputArea.n_cols); unpooledError.fill(error(rowidx, colidx) / OutputArea.n_elem); From faa76a0cf3305212dd1a0f90c0d216eb06ce234f Mon Sep 17 00:00:00 2001 From: Omar Shrit Date: Mon, 18 Sep 2023 20:23:47 +0200 Subject: [PATCH 3/4] Remove the serialization line, not needed in the main repo Signed-off-by: Omar Shrit --- src/mlpack/methods/ann/layer/serialization.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mlpack/methods/ann/layer/serialization.hpp b/src/mlpack/methods/ann/layer/serialization.hpp index 508ddbdaf0..a32c362c0c 100644 --- a/src/mlpack/methods/ann/layer/serialization.hpp +++ b/src/mlpack/methods/ann/layer/serialization.hpp @@ -76,6 +76,5 @@ CEREAL_REGISTER_TYPE(mlpack::FTSwishType<__VA_ARGS__>); \ CEREAL_REGISTER_MLPACK_LAYERS(arma::mat); -CEREAL_REGISTER_MLPACK_LAYERS(arma::fmat); #endif From 5875ab820874665ea90f621131f324c0fac55ea8 Mon Sep 17 00:00:00 2001 From: Omar Shrit Date: Tue, 19 Sep 2023 11:12:29 +0200 Subject: [PATCH 4/4] Remove explicit template parameters Signed-off-by: Omar Shrit --- .../methods/ann/layer/batch_norm_impl.hpp | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index 336c5c06fa..a2744048ec 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -216,26 +216,26 @@ void BatchNormType::Forward( // Calculate mean and variance over all channels. MatType mean = arma::sum(arma::sum(inputTemp, 2), 0) / m; variance = arma::sum(arma::sum(arma::pow( - inputTemp.each_slice() - arma::repmat(mean, + inputTemp.each_slice() - arma::repmat(mean, inputSize, 1), 2), 2), 0) / m; - outputTemp.each_slice() -= arma::repmat(mean, inputSize, 1); + outputTemp.each_slice() -= arma::repmat(mean, inputSize, 1); // Used in backward propagation. inputMean.set_size(arma::size(inputTemp)); inputMean = outputTemp; // Normalize output. - outputTemp.each_slice() /= arma::sqrt(arma::repmat(variance, + outputTemp.each_slice() /= arma::sqrt(arma::repmat(variance, inputSize, 1) + eps); // Re-used in backward propagation. normalized.set_size(arma::size(inputTemp)); normalized = outputTemp; - outputTemp.each_slice() %= arma::repmat(gamma.t(), + outputTemp.each_slice() %= arma::repmat(gamma.t(), inputSize, 1); - outputTemp.each_slice() += arma::repmat(beta.t(), + outputTemp.each_slice() += arma::repmat(beta.t(), inputSize, 1); count += 1; @@ -260,13 +260,13 @@ void BatchNormType::Forward( const_cast(output).memptr(), inputSize, size, batchSize * higherDimension, false, false); - outputTemp.each_slice() -= arma::repmat(runningMean.t(), + outputTemp.each_slice() -= arma::repmat(runningMean.t(), inputSize, 1); - outputTemp.each_slice() /= arma::sqrt(arma::repmat(runningVariance.t(), + outputTemp.each_slice() /= arma::sqrt(arma::repmat(runningVariance.t(), inputSize, 1) + eps); - outputTemp.each_slice() %= arma::repmat(gamma.t(), + outputTemp.each_slice() %= arma::repmat(gamma.t(), inputSize, 1); - outputTemp.each_slice() += arma::repmat(beta.t(), + outputTemp.each_slice() += arma::repmat(beta.t(), inputSize, 1); } } @@ -292,7 +292,7 @@ void BatchNormType::Backward( // Step 1: dl / dxhat. arma::Cube norm = - gyTemp.each_slice() % arma::repmat(gamma.t(), inputSize, 1); + gyTemp.each_slice() % arma::repmat(gamma.t(), inputSize, 1); // Step 2: sum dl / dxhat * (x - mu) * -0.5 * stdInv^3. MatType temp = arma::sum(arma::sum(norm % inputMean, 2), 0); @@ -300,17 +300,17 @@ void BatchNormType::Backward( // Step 3: dl / dxhat * 1 / stdInv + variance * 2 * (x - mu) / m + // dl / dmu * 1 / m. - gTemp = (norm.each_slice() % arma::repmat(stdInv, + gTemp = (norm.each_slice() % arma::repmat(stdInv, inputSize, 1)) + - ((inputMean.each_slice() % arma::repmat(vars, inputSize, 1) * 2.0) / m); + ((inputMean.each_slice() % arma::repmat(vars, inputSize, 1) * 2.0) / m); // Step 4: sum (dl / dxhat * -1 / stdInv) + variance * // sum (-2 * (x - mu)) / m. MatType normTemp = arma::sum(arma::sum((norm.each_slice() % - arma::repmat(-stdInv, inputSize, 1)) + - (inputMean.each_slice() % arma::repmat(vars, inputSize, 1) * (-2.0) / m), + arma::repmat(-stdInv, inputSize, 1)) + + (inputMean.each_slice() % arma::repmat(vars, inputSize, 1) * (-2.0) / m), 2), 0) / m; - gTemp.each_slice() += arma::repmat(normTemp, inputSize, 1); + gTemp.each_slice() += arma::repmat(normTemp, inputSize, 1); } template