From c84a4e510b98a17d8883b928871f315cb30b4cd7 Mon Sep 17 00:00:00 2001 From: iamshnoo Date: Tue, 11 Aug 2020 01:01:45 +0530 Subject: [PATCH 01/37] Pixel Shuffle layer. Commit 1. --- src/mlpack/methods/ann/layer/CMakeLists.txt | 2 + src/mlpack/methods/ann/layer/layer.hpp | 1 + src/mlpack/methods/ann/layer/layer_types.hpp | 2 + .../methods/ann/layer/pixel_shuffle.hpp | 179 ++++++++++++++++++ .../methods/ann/layer/pixel_shuffle_impl.hpp | 144 ++++++++++++++ src/mlpack/tests/ann_layer_test.cpp | 74 ++++++++ 6 files changed, 402 insertions(+) create mode 100644 src/mlpack/methods/ann/layer/pixel_shuffle.hpp create mode 100644 src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt index c3ae086c87..b4034f580c 100644 --- a/src/mlpack/methods/ann/layer/CMakeLists.txt +++ b/src/mlpack/methods/ann/layer/CMakeLists.txt @@ -79,6 +79,8 @@ set(SOURCES noisylinear_impl.hpp parametric_relu.hpp parametric_relu_impl.hpp + pixel_shuffle.hpp + pixel_shuffle_impl.hpp recurrent.hpp recurrent_impl.hpp recurrent_attention.hpp diff --git a/src/mlpack/methods/ann/layer/layer.hpp b/src/mlpack/methods/ann/layer/layer.hpp index 8e8e00691e..6673411616 100644 --- a/src/mlpack/methods/ann/layer/layer.hpp +++ b/src/mlpack/methods/ann/layer/layer.hpp @@ -55,6 +55,7 @@ #include "noisylinear.hpp" #include "padding.hpp" #include "parametric_relu.hpp" +#include "pixel_shuffle.hpp" #include "recurrent_attention.hpp" #include "recurrent.hpp" #include "reinforce_normal.hpp" diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 0f52a24df7..9e8dd0ce36 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -272,6 +273,7 @@ using LayerTypes = boost::variant< NoisyLinear*, Padding*, PReLU*, + PixelShuffle*, Softmax*, TransposedConvolution, NaiveConvolution, diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle.hpp new file mode 100644 index 0000000000..27dea9ba38 --- /dev/null +++ b/src/mlpack/methods/ann/layer/pixel_shuffle.hpp @@ -0,0 +1,179 @@ +/** + * @file methods/ann/layer/pixel_shuffle.hpp + * @author Anjishnu Mukherjee + * + * Definition of the PixelShuffle class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_HPP +#define MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * Implementation of the PixelShuffle layer. + * + * For more information, refer to the following paper, + * + * @code + * @article{Shi16, + * author = {Wenzhe Shi, Jose Caballero,Ferenc Huszár, Johannes Totz, + * Andrew P. Aitken, Rob Bishop, Daniel Rueckert, Zehan Wang}, + * title = {Real-Time Single Image and Video Super-Resolution Using an + * Efficient Sub-Pixel Convolutional Neural Network}, + * journal = {CoRR}, + * volume = {abs/1609.05158}, + * year = {2016}, + * url = {https://arxiv.org/abs/1609.05158}, + * eprint = {1609.05158}, + * } + * @endcode + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class PixelShuffle +{ + public: + //! Create the PixelShuffle object. + PixelShuffle(); + /** + * Create the PixelShuffle object using the specified parameters. + * The number of input channels should be an integral multiple of the square + * of the upscale factor. + * + * @param upscaleFactor The scaling factor for Pixel Shuffle. + * @param height The height of each input image. + * @param width The width of each input image. + * @param size The number of channels of each input image. + */ + PixelShuffle( size_t upscaleFactor, + size_t height, + size_t width, + size_t size); + + /** + * Ordinary feed forward pass of the PixelShuffle layer. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + template + void Forward(const arma::Mat& input, arma::Mat& output); + + /** + * Ordinary feed backward pass of the PixelShuffle layer. + * + * @param * (input) The propagated input activation. + * @param gy The backpropagated error. + * @param g The calculated gradient. + */ + template + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the upscale factor. + size_t UpscaleFactor() const { return upscaleFactor; } + + //! Modify the upscale factor. + size_t& UpscaleFactor() { return upscaleFactor; } + + //! Get the input image height. + size_t InputHeight() const { return height; } + + //! Modify the input image height. + size_t& InputHeight() { return height; } + + //! Get the input image width. + size_t InputWidth() const { return width; } + + //! Modify the input image width. + size_t& InputWidth() { return width; } + + //! Get the number of input channels. + size_t InputChannels() const { return size; } + + //! Modify the number of input channels. + size_t& InputChannels() { return size; } + + //! Get the output image height. + size_t OutputHeight() const { return outputHeight; } + + //! Get the output image width. + size_t OutputWidth() const { return outputWidth; } + + //! Get the number of output channels. + size_t OutputChannels() const { return sizeOut; } + + /** + * Serialize the layer. + */ + template + void serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! The scaling factor for Pixel Shuffle. + size_t upscaleFactor; + + //! The height of each input image. + size_t height; + + //! The width of each input image. + size_t width; + + //! The number of channels of each input image. + size_t size; + + //! The number of images in the batch. + size_t batchSize; + + //! The height of each output image. + size_t outputHeight; + + //! The width of each output image. + size_t outputWidth; + + //! The number of channels of each output image. + size_t sizeOut; + + //! A boolean used to do some internal calculations once initially. + bool reset; +}; // class PixelShuffle + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "pixel_shuffle_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp new file mode 100644 index 0000000000..cfbd4287b9 --- /dev/null +++ b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp @@ -0,0 +1,144 @@ +/** + * @file methods/ann/layer/pixel_shuffle_impl.hpp + * @author Anjishnu Mukherjee + * + * Implementation of the PixelShuffle class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_IMPL_HPP + +// In case it hasn't yet been included. +#include "pixel_shuffle.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +PixelShuffle::PixelShuffle() : + upscaleFactor(0), + height(0), + width(0), + size(0), + reset(false) +{ + // Nothing to do here. +} + +template +PixelShuffle::PixelShuffle( + size_t upscaleFactor, + size_t height, + size_t width, + size_t size) : + upscaleFactor(upscaleFactor), + height(height), + width(width), + size(size), + reset(false) +{ + // Nothing to do here. +} + +template +template +void PixelShuffle::Forward( + const arma::Mat& input, arma::Mat& output) +{ + if(!reset) + { + batchSize = input.n_cols; + sizeOut = size / std::pow(upscaleFactor, 2); + outputHeight = height * upscaleFactor; + outputWidth = width * upscaleFactor; + reset = true; + } + output.zeros(outputHeight * outputWidth * sizeOut, batchSize); + for(size_t n = 0; n < batchSize; n++) + { + arma::mat inputImage = input.col(n); + arma::mat outputImage = output.col(n); + arma::cube inputTemp(const_cast(inputImage).memptr(), height, + width, size, false, false); + arma::cube outputTemp(const_cast(outputImage).memptr(), + outputHeight, outputWidth, sizeOut, false, false); + + for (size_t c = 0; c < sizeOut ; c++) + { + for (size_t h = 0; h < outputHeight; h++) + { + for (size_t w = 0; w < outputWidth; w++) + { + size_t height_index = h / upscaleFactor; + size_t width_index = w / upscaleFactor; + size_t channel_index = (upscaleFactor * (h % upscaleFactor)) + + (w % upscaleFactor) + (c * std::pow(upscaleFactor, 2)); + outputTemp(w, h, c) = inputTemp(width_index, height_index, + channel_index); + } + } + } + output.col(n) = outputImage; + } +} + +template +template +void PixelShuffle::Backward( + const arma::Mat& input, const arma::Mat& gy, arma::Mat& g) +{ + g.zeros(arma::size(input)); + for(size_t n = 0; n < batchSize; n++) + { + arma::mat gyImage = gy.col(n); + arma::mat gImage = g.col(n); + arma::cube gyTemp(const_cast(gyImage).memptr(), outputHeight, + outputWidth, sizeOut, false, false); + arma::cube gTemp(const_cast(gImage).memptr(), height, width, + size, false, false); + + for (size_t c = 0; c < sizeOut ; c++) + { + for (size_t h = 0; h < outputHeight; h++) + { + for (size_t w = 0; w < outputWidth; w++) + { + size_t height_index = h / upscaleFactor; + size_t width_index = w / upscaleFactor; + size_t channel_index = (upscaleFactor * (h % upscaleFactor)) + + (w % upscaleFactor) + (c * std::pow(upscaleFactor, 2)); + gTemp(width_index, height_index, channel_index) = gyTemp(w, h, c); + } + } + } + + g.col(n) = gImage; + } +} + +template +template +void PixelShuffle::serialize( + Archive& ar, + const unsigned int /* version */) +{ + ar & BOOST_SERIALIZATION_NVP(delta); + ar & BOOST_SERIALIZATION_NVP(outputParameter); + ar & BOOST_SERIALIZATION_NVP(upscaleFactor); + ar & BOOST_SERIALIZATION_NVP(height); + ar & BOOST_SERIALIZATION_NVP(width); + ar & BOOST_SERIALIZATION_NVP(size); + ar & BOOST_SERIALIZATION_NVP(batchSize); + ar & BOOST_SERIALIZATION_NVP(outputHeight); + ar & BOOST_SERIALIZATION_NVP(outputWidth); + ar & BOOST_SERIALIZATION_NVP(sizeOut); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 29e16273b8..38fec13c05 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -4186,3 +4186,77 @@ TEST_CASE("BatchNormDeterministicTest", "[ANNLayerTest]") // The model should switch to training mode for predicting. REQUIRE(boost::get*>(module.Model()[0])->Deterministic() == 0); } + +/** + * Simple Test for PixelShuffle layer. + */ +TEST_CASE("PixelShuffleLayerTest", "[ANNLayerTest]") +{ + arma::mat input, output, gy, g, outputExpected, gExpected; + PixelShuffle<> module(2, 2, 2, 4); + + // Input is a batch of 2 images, each of size (2,2) and having 4 channels. + input << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 + << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0 + << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr; + + gy << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8 + << 12 << 16 << arma::endr << 17 << 21 << 25 << 29 << 18 << 22 << 26 << 30 + << 19 << 23 << 27 << 31 << 20 << 24 << 28 << 32 << arma::endr; + + // Calculated using torch.nn.PixelShuffle(). + outputExpected << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0 + << 0 << 0 << 0 << 0 << arma::endr << 5 << 0 << 7 << 0 << 0 << 0 << 0 << 0 + << 6 << 0 << 8 << 0 << 0 << 0 << 0 << 0 << arma::endr; + gExpected << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12 + << 6 << 14 << 8 << 16 << arma::endr << 17 << 25 << 19 << 27 << 21 << 29 + << 23 << 31 << 18 << 26 << 20 << 28 << 22 << 30 << 24 << 32 << arma::endr; + + input = input.t(); + outputExpected = outputExpected.t(); + gy = gy.t(); + gExpected = gExpected.t(); + + // Check the Forward pass of the layer. + module.Forward(input, output); + CheckMatrices(output, outputExpected); + + // Check the Backward pass of the layer. + module.Backward(input, gy, g); + CheckMatrices(g, gExpected); +} + +/** + * Test that the function that can access the parameters of the + * PixelShuffle layer works. + */ +TEST_CASE("PixelShuffleLayerParametersTest", "[ANNLayerTest]") +{ + // Create the layer using the empty constructor. + PixelShuffle<> layer; + + // Set the different input parameters of the layer. + layer.UpscaleFactor() = 2; + layer.InputHeight() = 2; + layer.InputWidth() = 2; + layer.InputChannels() = 4; + + // Make sure we can get the parameters successfully. + REQUIRE(layer.UpscaleFactor() == 2); + REQUIRE(layer.InputHeight() == 2); + REQUIRE(layer.InputWidth() == 2); + REQUIRE(layer.InputChannels() == 4); + + arma::mat input, output; + // Input is a batch of 2 images, each of size (2,2) and having 4 channels. + input << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 + << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0 + << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr; + input = input.t(); + layer.Forward(input, output); + + // Check whether output parameters are returned correctly. + REQUIRE(layer.OutputHeight() == 4 ); + REQUIRE(layer.OutputWidth() == 4); + REQUIRE(layer.OutputChannels() == 1); +} From 64dd26145a1b1c02593c105a10e2622d1aa33634 Mon Sep 17 00:00:00 2001 From: iamshnoo Date: Fri, 14 Aug 2020 19:08:50 +0530 Subject: [PATCH 02/37] Fix style issues. --- src/mlpack/methods/ann/layer/pixel_shuffle.hpp | 10 +++++----- src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle.hpp index 27dea9ba38..2fda7cf5f4 100644 --- a/src/mlpack/methods/ann/layer/pixel_shuffle.hpp +++ b/src/mlpack/methods/ann/layer/pixel_shuffle.hpp @@ -60,10 +60,10 @@ class PixelShuffle * @param width The width of each input image. * @param size The number of channels of each input image. */ - PixelShuffle( size_t upscaleFactor, - size_t height, - size_t width, - size_t size); + PixelShuffle(size_t upscaleFactor, + size_t height, + size_t width, + size_t size); /** * Ordinary feed forward pass of the PixelShuffle layer. @@ -77,7 +77,7 @@ class PixelShuffle /** * Ordinary feed backward pass of the PixelShuffle layer. * - * @param * (input) The propagated input activation. + * @param input The propagated input activation. * @param gy The backpropagated error. * @param g The calculated gradient. */ diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp index cfbd4287b9..3ca2852074 100644 --- a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp +++ b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp @@ -49,7 +49,7 @@ template void PixelShuffle::Forward( const arma::Mat& input, arma::Mat& output) { - if(!reset) + if (!reset) { batchSize = input.n_cols; sizeOut = size / std::pow(upscaleFactor, 2); @@ -58,7 +58,7 @@ void PixelShuffle::Forward( reset = true; } output.zeros(outputHeight * outputWidth * sizeOut, batchSize); - for(size_t n = 0; n < batchSize; n++) + for (size_t n = 0; n < batchSize; n++) { arma::mat inputImage = input.col(n); arma::mat outputImage = output.col(n); @@ -92,7 +92,7 @@ void PixelShuffle::Backward( const arma::Mat& input, const arma::Mat& gy, arma::Mat& g) { g.zeros(arma::size(input)); - for(size_t n = 0; n < batchSize; n++) + for (size_t n = 0; n < batchSize; n++) { arma::mat gyImage = gy.col(n); arma::mat gImage = g.col(n); From 5da525cde653ee3ed2bd5c4c979bdacdbd9b7959 Mon Sep 17 00:00:00 2001 From: iamshnoo Date: Fri, 14 Aug 2020 23:49:48 +0530 Subject: [PATCH 03/37] Fix consistency issue for constructor format. --- src/mlpack/methods/ann/layer/pixel_shuffle.hpp | 8 ++++---- src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle.hpp index 2fda7cf5f4..c62b0f0ecb 100644 --- a/src/mlpack/methods/ann/layer/pixel_shuffle.hpp +++ b/src/mlpack/methods/ann/layer/pixel_shuffle.hpp @@ -60,10 +60,10 @@ class PixelShuffle * @param width The width of each input image. * @param size The number of channels of each input image. */ - PixelShuffle(size_t upscaleFactor, - size_t height, - size_t width, - size_t size); + PixelShuffle(const size_t upscaleFactor, + const size_t height, + const size_t width, + const size_t size); /** * Ordinary feed forward pass of the PixelShuffle layer. diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp index 3ca2852074..d65a0c456d 100644 --- a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp +++ b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp @@ -31,10 +31,10 @@ PixelShuffle::PixelShuffle() : template PixelShuffle::PixelShuffle( - size_t upscaleFactor, - size_t height, - size_t width, - size_t size) : + const size_t upscaleFactor, + const size_t height, + const size_t width, + const size_t size) : upscaleFactor(upscaleFactor), height(height), width(width), From 438cc009898dad31326636af9f3d47d908ced4af Mon Sep 17 00:00:00 2001 From: iamshnoo Date: Tue, 25 Aug 2020 11:47:28 +0530 Subject: [PATCH 04/37] Use suggestions from code review. --- .../methods/ann/layer/pixel_shuffle_impl.hpp | 1 - src/mlpack/tests/ann_layer_test.cpp | 56 ++++++++++++++----- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp index d65a0c456d..31d137eb59 100644 --- a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp +++ b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp @@ -115,7 +115,6 @@ void PixelShuffle::Backward( } } } - g.col(n) = gImage; } } diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index c3c0b11edc..a1ed47db97 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -4302,38 +4302,64 @@ TEST_CASE("TransposedConvolutionWeightInitializationTest", "[ANNLayerTest]") */ TEST_CASE("PixelShuffleLayerTest", "[ANNLayerTest]") { - arma::mat input, output, gy, g, outputExpected, gExpected; - PixelShuffle<> module(2, 2, 2, 4); + arma::mat input1, output1, gy1, g1, outputExpected1, gExpected1; + arma::mat input2, output2, gy2, g2, outputExpected2, gExpected2; + PixelShuffle<> module1(2, 2, 2, 4); + PixelShuffle<> module2(2, 2, 2, 4); + + // Input is a single image, of size (2,2) and having 4 channels. + input1 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 + << 0 << 0 << arma::endr; + gy1 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8 + << 12 << 16 << arma::endr; + + // Calculated using torch.nn.PixelShuffle(). + outputExpected1 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0 + << 0 << 0 << 0 << 0 << arma::endr; + gExpected1 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12 + << 6 << 14 << 8 << 16 << arma::endr; + + input1 = input1.t(); + outputExpected1 = outputExpected1.t(); + gy1 = gy1.t(); + gExpected1 = gExpected1.t(); + + // Check the Forward pass of the layer. + module1.Forward(input1, output1); + CheckMatrices(output1, outputExpected1); + + // Check the Backward pass of the layer. + module1.Backward(input1, gy1, g1); + CheckMatrices(g1, gExpected1); // Input is a batch of 2 images, each of size (2,2) and having 4 channels. - input << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 + input2 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr; - - gy << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8 + gy2 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8 << 12 << 16 << arma::endr << 17 << 21 << 25 << 29 << 18 << 22 << 26 << 30 << 19 << 23 << 27 << 31 << 20 << 24 << 28 << 32 << arma::endr; // Calculated using torch.nn.PixelShuffle(). - outputExpected << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0 + outputExpected2 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0 << 0 << 0 << 0 << 0 << arma::endr << 5 << 0 << 7 << 0 << 0 << 0 << 0 << 0 << 6 << 0 << 8 << 0 << 0 << 0 << 0 << 0 << arma::endr; - gExpected << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12 + gExpected2 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12 << 6 << 14 << 8 << 16 << arma::endr << 17 << 25 << 19 << 27 << 21 << 29 << 23 << 31 << 18 << 26 << 20 << 28 << 22 << 30 << 24 << 32 << arma::endr; - input = input.t(); - outputExpected = outputExpected.t(); - gy = gy.t(); - gExpected = gExpected.t(); + input2 = input2.t(); + outputExpected2 = outputExpected2.t(); + gy2 = gy2.t(); + gExpected2 = gExpected2.t(); // Check the Forward pass of the layer. - module.Forward(input, output); - CheckMatrices(output, outputExpected); + module2.Forward(input2, output2); + CheckMatrices(output2, outputExpected2); // Check the Backward pass of the layer. - module.Backward(input, gy, g); - CheckMatrices(g, gExpected); + module2.Backward(input2, gy2, g2); + CheckMatrices(g2, gExpected2); } /** From c6cd6b860c3e56c76c2df9abe0bfe17c3bf7d1e1 Mon Sep 17 00:00:00 2001 From: iamshnoo Date: Tue, 25 Aug 2020 16:25:52 +0530 Subject: [PATCH 05/37] FIx static analysis issue. --- src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp index 31d137eb59..796ee54071 100644 --- a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp +++ b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp @@ -24,6 +24,10 @@ PixelShuffle::PixelShuffle() : height(0), width(0), size(0), + batchSize(0), + outputHeight(0), + outputWidth(0), + sizeOut(0), reset(false) { // Nothing to do here. @@ -39,6 +43,10 @@ PixelShuffle::PixelShuffle( height(height), width(width), size(size), + batchSize(0), + outputHeight(0), + outputWidth(0), + sizeOut(0), reset(false) { // Nothing to do here. From cbb151000528700e9c57933030fc71e4c0803b70 Mon Sep 17 00:00:00 2001 From: iamshnoo Date: Wed, 26 Aug 2020 09:46:37 +0530 Subject: [PATCH 06/37] Update HISTORY.md for Pixel Shuffle layer. --- HISTORY.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index c04b2aeca1..ba5bb0b194 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,7 @@ ### mlpack ?.?.? ###### ????-??-?? + * Added Pixel Shuffle layer (#2563). + * Force CMake to show error when it didn't find Python/modules (#2568). * Refactor `ProgramInfo()` to separate out all the different From 709eb40dbe731a6baab317c4c2361e540e78a02d Mon Sep 17 00:00:00 2001 From: Abhinav Anand Date: Tue, 9 Feb 2021 02:45:23 +0530 Subject: [PATCH 07/37] minor --- src/mlpack/methods/ann/layer/kmax_pooling.hpp | 310 ++++++++++++++++++ .../methods/ann/layer/kmax_pooling_impl.hpp | 165 ++++++++++ 2 files changed, 475 insertions(+) create mode 100644 src/mlpack/methods/ann/layer/kmax_pooling.hpp create mode 100644 src/mlpack/methods/ann/layer/kmax_pooling_impl.hpp diff --git a/src/mlpack/methods/ann/layer/kmax_pooling.hpp b/src/mlpack/methods/ann/layer/kmax_pooling.hpp new file mode 100644 index 0000000000..098a9d100a --- /dev/null +++ b/src/mlpack/methods/ann/layer/kmax_pooling.hpp @@ -0,0 +1,310 @@ +/** + * @file methods/ann/layer/max_pooling.hpp + * @author Marcus Edel + * @author Nilay Jain + * + * Definition of the MaxPooling class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_MAX_POOLING_HPP +#define MLPACK_METHODS_ANN_LAYER_MAX_POOLING_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/* + * The max pooling rule for convolution neural networks. Take the maximum value + * within the receptive block. + */ +class MaxPoolingRule +{ + public: + /* + * Return the maximum value within the receptive block. + * + * @param input Input used to perform the pooling operation. + */ + template + size_t Pooling(const MatType& input) + { + return arma::as_scalar(arma::find(input.max() == input, 1)); + } +}; + +/** + * Implementation of the MaxPooling layer. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class MaxPooling +{ + public: + //! Create the MaxPooling object. + MaxPooling(); + + /** + * Create the MaxPooling object using the specified number of units. + * + * @param kernelWidth Width of the pooling window. + * @param kernelHeight Height of the pooling window. + * @param strideWidth Width of the stride operation. + * @param strideHeight Width of the stride operation. + * @param floor Rounding operator (floor or ceil). + */ + MaxPooling(const size_t kernelWidth, + const size_t kernelHeight, + const size_t strideWidth = 1, + const size_t strideHeight = 1, + const bool floor = true); + + /** + * Ordinary feed forward pass of a neural network, evaluating the function + * f(x) by propagating the activity forward through f. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + template + void Forward(const arma::Mat& input, arma::Mat& output); + + /** + * Ordinary feed backward pass of a neural network, using 3rd-order tensors as + * input, calculating the function f(x) by propagating x backwards through f. + * Using the results from the feed forward pass. + * + * @param * (input) The propagated input activation. + * @param gy The backpropagated error. + * @param g The calculated gradient. + */ + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); + + //! Get the output parameter. + const OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + const OutputDataType& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the input width. + size_t InputWidth() const { return inputWidth; } + //! Modify the input width. + size_t& InputWidth() { return inputWidth; } + + //! Get the input height. + size_t InputHeight() const { return inputHeight; } + //! Modify the input height. + size_t& InputHeight() { return inputHeight; } + + //! Get the output width. + size_t OutputWidth() const { return outputWidth; } + //! Modify the output width. + size_t& OutputWidth() { return outputWidth; } + + //! Get the output height. + size_t OutputHeight() const { return outputHeight; } + //! Modify the output height. + size_t& OutputHeight() { return outputHeight; } + + //! Get the input size. + size_t InputSize() const { return inSize; } + + //! Get the output size. + size_t OutputSize() const { return outSize; } + + //! Get the kernel width. + size_t KernelWidth() const { return kernelWidth; } + //! Modify the kernel width. + size_t& KernelWidth() { return kernelWidth; } + + //! Get the kernel height. + size_t KernelHeight() const { return kernelHeight; } + //! Modify the kernel height. + size_t& KernelHeight() { return kernelHeight; } + + //! Get the stride width. + size_t StrideWidth() const { return strideWidth; } + //! Modify the stride width. + size_t& StrideWidth() { return strideWidth; } + + //! Get the stride height. + size_t StrideHeight() const { return strideHeight; } + //! Modify the stride height. + size_t& StrideHeight() { return strideHeight; } + + //! Get the value of the rounding operation. + bool Floor() const { return floor; } + //! Modify the value of the rounding operation. + bool& Floor() { return floor; } + + //! Get the value of the deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of the deterministic parameter. + bool& Deterministic() { return deterministic; } + + //! Get the size of the weights. + size_t WeightSize() const { return 0; } + + /** + * Serialize the layer. + */ + template + void serialize(Archive& ar, const uint32_t /* version */); + + private: + /** + * Apply pooling to the input and store the results. + * + * @param input The input to be apply the pooling rule. + * @param output The pooled result. + * @param poolingIndices The pooled indices. + */ + template + void PoolingOperation(const arma::Mat& input, + arma::Mat& output, + arma::Mat& poolingIndices) + { + for (size_t j = 0, colidx = 0; j < output.n_cols; + ++j, colidx += strideHeight) + { + for (size_t i = 0, rowidx = 0; i < output.n_rows; + ++i, rowidx += strideWidth) + { + arma::mat subInput = input( + arma::span(rowidx, rowidx + kernelWidth - 1 - offset), + arma::span(colidx, colidx + kernelHeight - 1 - offset)); + + const size_t idx = pooling.Pooling(subInput); + output(i, j) = subInput(idx); + + if (!deterministic) + { + arma::Mat subIndices = indices(arma::span(rowidx, + rowidx + kernelWidth - 1 - offset), + arma::span(colidx, colidx + kernelHeight - 1 - offset)); + + poolingIndices(i, j) = subIndices(idx); + } + } + } + } + + /** + * Apply unpooling to the input and store the results. + * + * @param error The backward error. + * @param output The pooled result. + * @param poolingIndices The pooled indices. + */ + template + void Unpooling(const arma::Mat& error, + arma::Mat& output, + arma::Mat& poolingIndices) + { + for (size_t i = 0; i < poolingIndices.n_elem; ++i) + { + output(poolingIndices(i)) += error(i); + } + } + + //! Locally-stored width of the pooling window. + size_t kernelWidth; + + //! Locally-stored height of the pooling window. + size_t kernelHeight; + + //! Locally-stored width of the stride operation. + size_t strideWidth; + + //! Locally-stored height of the stride operation. + size_t strideHeight; + + //! Rounding operation used. + bool floor; + + //! Locally-stored number of input channels. + size_t inSize; + + //! Locally-stored number of output channels. + size_t outSize; + + //! Locally-stored reset parameter used to initialize the module once. + bool reset; + + //! Locally-stored input width. + size_t inputWidth; + + //! Locally-stored input height. + size_t inputHeight; + + //! Locally-stored output width. + size_t outputWidth; + + //! Locally-stored output height. + size_t outputHeight; + + //! If true use maximum a posteriori during the forward pass. + bool deterministic; + + //! Locally-stored stored rounding offset. + size_t offset; + + //! Locally-stored number of input units. + size_t batchSize; + + //! Locally-stored output parameter. + arma::cube outputTemp; + + //! Locally-stored transformed input parameter. + arma::cube inputTemp; + + //! Locally-stored transformed output parameter. + arma::cube gTemp; + + //! Locally-stored pooling strategy. + MaxPoolingRule pooling; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Locally-stored indices matrix parameter. + arma::Mat indices; + + //! Locally-stored indices column parameter. + arma::Col indicesCol; + + //! Locally-stored pooling indicies. + std::vector poolingIndices; +}; // class MaxPooling + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "max_pooling_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/layer/kmax_pooling_impl.hpp b/src/mlpack/methods/ann/layer/kmax_pooling_impl.hpp new file mode 100644 index 0000000000..cbc17904c4 --- /dev/null +++ b/src/mlpack/methods/ann/layer/kmax_pooling_impl.hpp @@ -0,0 +1,165 @@ +/** + * @file methods/ann/layer/max_pooling_impl.hpp + * @author Marcus Edel + * @author Nilay Jain + * + * Implementation of the MaxPooling class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_MAX_POOLING_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_MAX_POOLING_IMPL_HPP + +// In case it hasn't yet been included. +#include "max_pooling.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +MaxPooling::MaxPooling() +{ + // Nothing to do here. +} + +template +MaxPooling::MaxPooling( + const size_t kernelWidth, + const size_t kernelHeight, + const size_t strideWidth, + const size_t strideHeight, + const bool floor) : + kernelWidth(kernelWidth), + kernelHeight(kernelHeight), + strideWidth(strideWidth), + strideHeight(strideHeight), + floor(floor), + inSize(0), + outSize(0), + reset(false), + inputWidth(0), + inputHeight(0), + outputWidth(0), + outputHeight(0), + deterministic(false), + offset(0), + batchSize(0) +{ + // Nothing to do here. +} + +template +template +void MaxPooling::Forward( + const arma::Mat& input, arma::Mat& output) +{ + batchSize = input.n_cols; + inSize = input.n_elem / (inputWidth * inputHeight * batchSize); + inputTemp = arma::cube(const_cast&>(input).memptr(), + inputWidth, inputHeight, batchSize * inSize, false, false); + + if (floor) + { + outputWidth = std::floor((inputWidth - + (double) kernelWidth) / (double) strideWidth + 1); + outputHeight = std::floor((inputHeight - + (double) kernelHeight) / (double) strideHeight + 1); + offset = 0; + } + else + { + outputWidth = std::ceil((inputWidth - + (double) kernelWidth) / (double) strideWidth + 1); + outputHeight = std::ceil((inputHeight - + (double) kernelHeight) / (double) strideHeight + 1); + offset = 1; + } + + outputTemp = arma::zeros >(outputWidth, outputHeight, + batchSize * inSize); + + if (!deterministic) + { + poolingIndices.push_back(outputTemp); + } + + if (!reset) + { + size_t elements = inputWidth * inputHeight; + indicesCol = arma::linspace >(0, (elements - 1), + elements); + + indices = arma::Mat(indicesCol.memptr(), inputWidth, inputHeight); + + reset = true; + } + + for (size_t s = 0; s < inputTemp.n_slices; s++) + { + if (!deterministic) + { + PoolingOperation(inputTemp.slice(s), outputTemp.slice(s), + poolingIndices.back().slice(s)); + } + else + { + PoolingOperation(inputTemp.slice(s), outputTemp.slice(s), + inputTemp.slice(s)); + } + } + + output = arma::Mat(outputTemp.memptr(), outputTemp.n_elem / batchSize, + batchSize); + + outputWidth = outputTemp.n_rows; + outputHeight = outputTemp.n_cols; + outSize = batchSize * inSize; +} + +template +template +void MaxPooling::Backward( + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) +{ + arma::cube mappedError = arma::cube(((arma::Mat&) gy).memptr(), + outputWidth, outputHeight, outSize, false, false); + + gTemp = arma::zeros(inputTemp.n_rows, + inputTemp.n_cols, inputTemp.n_slices); + + for (size_t s = 0; s < mappedError.n_slices; s++) + { + Unpooling(mappedError.slice(s), gTemp.slice(s), + poolingIndices.back().slice(s)); + } + + poolingIndices.pop_back(); + + g = arma::mat(gTemp.memptr(), gTemp.n_elem / batchSize, batchSize); +} + +template +template +void MaxPooling::serialize( + Archive& ar, + const uint32_t /* version */) +{ + ar(CEREAL_NVP(kernelWidth)); + ar(CEREAL_NVP(kernelHeight)); + ar(CEREAL_NVP(strideWidth)); + ar(CEREAL_NVP(strideHeight)); + ar(CEREAL_NVP(batchSize)); + ar(CEREAL_NVP(floor)); + ar(CEREAL_NVP(inputWidth)); + ar(CEREAL_NVP(inputHeight)); + ar(CEREAL_NVP(outputWidth)); + ar(CEREAL_NVP(outputHeight)); +} + +} // namespace ann +} // namespace mlpack + +#endif From 03115231bc956588ad0815ea44000c5052a8f78a Mon Sep 17 00:00:00 2001 From: Abhinav Anand Date: Tue, 9 Feb 2021 03:47:23 +0530 Subject: [PATCH 08/37] Implemented ISRLU Activation Function --- src/mlpack/methods/ann/layer/CMakeLists.txt | 2 + src/mlpack/methods/ann/layer/isrlu.hpp | 145 ++++++++++++++++++ src/mlpack/methods/ann/layer/isrlu_impl.hpp | 70 +++++++++ src/mlpack/methods/ann/layer/layer_types.hpp | 2 + .../tests/activation_functions_test.cpp | 64 ++++++++ 5 files changed, 283 insertions(+) create mode 100644 src/mlpack/methods/ann/layer/isrlu.hpp create mode 100644 src/mlpack/methods/ann/layer/isrlu_impl.hpp diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt index b4726b0c6f..e7894b140b 100644 --- a/src/mlpack/methods/ann/layer/CMakeLists.txt +++ b/src/mlpack/methods/ann/layer/CMakeLists.txt @@ -46,6 +46,8 @@ set(SOURCES hard_tanh_impl.hpp highway.hpp highway_impl.hpp + isrlu.hpp + isrlu_impl.hpp join.hpp join_impl.hpp layer.hpp diff --git a/src/mlpack/methods/ann/layer/isrlu.hpp b/src/mlpack/methods/ann/layer/isrlu.hpp new file mode 100644 index 0000000000..d8fad5be47 --- /dev/null +++ b/src/mlpack/methods/ann/layer/isrlu.hpp @@ -0,0 +1,145 @@ +/** + * @file methods/ann/layer/isrlu.hpp + * @author Abhinav Anand + * + * Definition of the ISRLU activation function as described by Jonathan T. Barron. + * + * For more information, read the following paper. + * + * @code + * @article{ + * author = {Carlile, Brad and Delamarter, Guy and Kinney, Paul and Marti, Akiko and Whitney, Brian}, + * title = {Improving deep learning by inverse square root linear units (ISRLUs)}, + * year = {2017}, + * url = {https://arxiv.org/pdf/1710.09967.pdf} + * } + * @endcode + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_ISRLU_HPP +#define MLPACK_METHODS_ANN_LAYER_ISRLU_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * The ISRLU activation function, defined by + * + * @f{eqnarray*}{ + * f(x) &=& \left\{ + * \begin{array}{lr} + * x & : x \ge 0 \\ + * x(\frac{1}{1 + \alpha x^2}) & : x < 0 + * \end{array} + * \right. \\ + * f'(x) &=& \left\{ + * \begin{array}{lr} + * x & : 1 \ge 0 \\ + * (\frac{1}{1 + \alpha x^2})^3 & : x < 0 + * \end{array} + * \right. + * @f} + * + * In the deterministic mode, there is no computation of the derivative. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class ISRLU +{ + public: + /** + * Create the ISRLU object using the specified parameter. + * + * @param alpha Scale parameter controls the value to which an ISRLU + * saturates for negative inputs. + */ + ISRLU(const double alpha = 1.0); + + /** + * Ordinary feed forward pass of a neural network, evaluating the function + * f(x) by propagating the activity forward through f. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + template + void Forward(const InputType& input, OutputType& output); + + /** + * Ordinary feed backward pass of a neural network, calculating the function + * f(x) by propagating x backwards through f. Using the results from the feed + * forward pass. + * + * @param input The propagated input activation f(x). + * @param gy The backpropagated error. + * @param g The calculated gradient. + */ + template + void Backward(const DataType& input, const DataType& gy, DataType& g); + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the non zero gradient. + double const& Alpha() const { return alpha; } + //! Modify the non zero gradient. + double& Alpha() { return alpha; } + + //! Get the value of deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of deterministic parameter. + bool& Deterministic() { return deterministic; } + + //! Get size of weights. + size_t WeightSize() { return 0; } + + /** + * Serialize the layer. + */ + template + void serialize(Archive& ar, const uint32_t /* version */); + + private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Locally stored first derivative of the activation function. + arma::mat derivative; + + //! ISRLU Hyperparameter (alpha > 0). + double alpha; + + //! If true the derivative computation is disabled, see notes above. + bool deterministic; +}; // class ISRLU + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "isrlu_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/layer/isrlu_impl.hpp b/src/mlpack/methods/ann/layer/isrlu_impl.hpp new file mode 100644 index 0000000000..2774f6bb12 --- /dev/null +++ b/src/mlpack/methods/ann/layer/isrlu_impl.hpp @@ -0,0 +1,70 @@ +/** + * @file methods/ann/layer/isrlu_impl.hpp + * @author Gaurav Singh + * + * Implementation of the ISRLU activation function as described by Jonathan T. Barron. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_ISRLU_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_V_IMPL_HPP + +// In case it hasn't yet been included. +#include "isrlu.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +ISRLU::ISRLU(const double alpha) : + alpha(alpha), + deterministic(false) +{} + +template +template +void ISRLU::Forward( + const InputType& input, OutputType& output) +{ + output = arma::ones(arma::size(input)); + for (size_t i = 0; i < input.n_elem; ++i) + { + output(i) = (input(i) >= 0) ? input(i) : input(i) * + (1 / std::sqrt(1 + alpha*input(i)*input(i))); + } + + if (!deterministic) + { + derivative.set_size(arma::size(input)); + for (size_t i = 0; i < input.n_elem; ++i) + { + derivative(i) = (input(i) >= 0) ? 1 : + std:pow(1 / std::sqrt(1 + alpha*input(i)*input(i)), 3); + } + } +} + +template +template +void ISRLU::Backward( + const DataType& /* input */, const DataType& gy, DataType& g) +{ + g = gy % derivative; +} + +template +template +void ISRLU::serialize( + Archive& ar, + const uint32_t /* version */) +{ + ar(CEREAL_NVP(alpha)); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 1d7fd0ccba..72c3df52ce 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -52,6 +52,7 @@ #include #include #include +#include #include #include @@ -272,6 +273,7 @@ using LayerTypes = boost::variant< FlexibleReLU*, GRU*, HardTanH*, + ISRLU*, Join*, LayerNorm*, LeakyReLU*, diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index d88c0a5109..3ac6437cd0 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -558,6 +558,54 @@ void CheckCELUDerivativeCorrect(const arma::colvec input, } } +/** + * Implementation of the ISRLU activation function test. The function is + * implemented as ISRLU layer in the file isrlu.hpp. + * + * @param input Input data used for evaluating the ISRLU activation function. + * @param target Target data used to evaluate the ISRLU activation. + */ +void CheckISRLUActivationCorrect(const arma::colvec input, + const arma::colvec target) +{ + // Initialize ISRLU object with alpha = 1.0. + ISRLU<> lrf(1.0); + + // Test the activation function using the entire vector as input. + arma::colvec activations; + lrf.Forward(input, activations); + for (size_t i = 0; i < activations.n_elem; ++i) + { + REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +} + +/** + * Implementation of the ISRLU activation function derivative test. The function + * is implemented as ISRLU layer in the file isrlu.hpp. + * + * @param input Input data used for evaluating the ISRLU activation function. + * @param target Target data used to evaluate the ISRLU activation. + */ +void CheckISRLUDerivativeCorrect(const arma::colvec input, + const arma::colvec target) +{ + // Initialize ISRLU object with alpha = 1.0. + ISRLU<> lrf(1.0); + + // Test the calculation of the derivatives using the entire vector as input. + arma::colvec derivatives, activations; + + // This error vector will be set to 1 to get the derivatives. + arma::colvec error = arma::ones(input.n_elem); + lrf.Forward(input, activations); + lrf.Backward(activations, error, derivatives); + for (size_t i = 0; i < derivatives.n_elem; ++i) + { + REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); + } +} + /** * Implementation of the Softmin activation function test. The function is * implemented as Softmin layer in the file softmin.hpp. @@ -991,6 +1039,22 @@ TEST_CASE("CELUFunctionTest", "[ActivationFunctionsTest]") CheckCELUDerivativeCorrect(desiredActivations, desiredDerivatives); } +/** + * Basic test of the ISRLU activation function. + */ +TEST_CASE("ISRLUFunctionTest", "[ActivationFunctionsTest]") +{ + const arma::colvec desiredActivations("-0.89442719 3.2 4.5 \ + -0.99995020 1 -0.70710678 2 0"); + + const arma::colvec desiredDerivatives("0.74535599 1 1 \ + 0.70712438 1 \ + 0.81649658 1 1"); + + CheckISRLUActivationCorrect(activationData, desiredActivations); + CheckISRLUDerivativeCorrect(desiredActivations, desiredDerivatives); +} + /** * Basic test of the inverse quadratic function. */ From 638be09256da8c3e93e54c06c5f9b0e0cbcec61b Mon Sep 17 00:00:00 2001 From: Abhinav Anand Date: Tue, 9 Feb 2021 03:58:31 +0530 Subject: [PATCH 09/37] minor change --- src/mlpack/methods/ann/layer/isrlu_impl.hpp | 2 +- src/mlpack/methods/ann/layer/kmax_pooling.hpp | 310 ------------------ .../methods/ann/layer/kmax_pooling_impl.hpp | 165 ---------- 3 files changed, 1 insertion(+), 476 deletions(-) delete mode 100644 src/mlpack/methods/ann/layer/kmax_pooling.hpp delete mode 100644 src/mlpack/methods/ann/layer/kmax_pooling_impl.hpp diff --git a/src/mlpack/methods/ann/layer/isrlu_impl.hpp b/src/mlpack/methods/ann/layer/isrlu_impl.hpp index 2774f6bb12..8e74241305 100644 --- a/src/mlpack/methods/ann/layer/isrlu_impl.hpp +++ b/src/mlpack/methods/ann/layer/isrlu_impl.hpp @@ -1,6 +1,6 @@ /** * @file methods/ann/layer/isrlu_impl.hpp - * @author Gaurav Singh + * @author Abhinav Anand * * Implementation of the ISRLU activation function as described by Jonathan T. Barron. * diff --git a/src/mlpack/methods/ann/layer/kmax_pooling.hpp b/src/mlpack/methods/ann/layer/kmax_pooling.hpp deleted file mode 100644 index 098a9d100a..0000000000 --- a/src/mlpack/methods/ann/layer/kmax_pooling.hpp +++ /dev/null @@ -1,310 +0,0 @@ -/** - * @file methods/ann/layer/max_pooling.hpp - * @author Marcus Edel - * @author Nilay Jain - * - * Definition of the MaxPooling class. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_MAX_POOLING_HPP -#define MLPACK_METHODS_ANN_LAYER_MAX_POOLING_HPP - -#include - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -/* - * The max pooling rule for convolution neural networks. Take the maximum value - * within the receptive block. - */ -class MaxPoolingRule -{ - public: - /* - * Return the maximum value within the receptive block. - * - * @param input Input used to perform the pooling operation. - */ - template - size_t Pooling(const MatType& input) - { - return arma::as_scalar(arma::find(input.max() == input, 1)); - } -}; - -/** - * Implementation of the MaxPooling layer. - * - * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, - * arma::sp_mat or arma::cube). - * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, - * arma::sp_mat or arma::cube). - */ -template < - typename InputDataType = arma::mat, - typename OutputDataType = arma::mat -> -class MaxPooling -{ - public: - //! Create the MaxPooling object. - MaxPooling(); - - /** - * Create the MaxPooling object using the specified number of units. - * - * @param kernelWidth Width of the pooling window. - * @param kernelHeight Height of the pooling window. - * @param strideWidth Width of the stride operation. - * @param strideHeight Width of the stride operation. - * @param floor Rounding operator (floor or ceil). - */ - MaxPooling(const size_t kernelWidth, - const size_t kernelHeight, - const size_t strideWidth = 1, - const size_t strideHeight = 1, - const bool floor = true); - - /** - * Ordinary feed forward pass of a neural network, evaluating the function - * f(x) by propagating the activity forward through f. - * - * @param input Input data used for evaluating the specified function. - * @param output Resulting output activation. - */ - template - void Forward(const arma::Mat& input, arma::Mat& output); - - /** - * Ordinary feed backward pass of a neural network, using 3rd-order tensors as - * input, calculating the function f(x) by propagating x backwards through f. - * Using the results from the feed forward pass. - * - * @param * (input) The propagated input activation. - * @param gy The backpropagated error. - * @param g The calculated gradient. - */ - template - void Backward(const arma::Mat& /* input */, - const arma::Mat& gy, - arma::Mat& g); - - //! Get the output parameter. - const OutputDataType& OutputParameter() const { return outputParameter; } - //! Modify the output parameter. - OutputDataType& OutputParameter() { return outputParameter; } - - //! Get the delta. - const OutputDataType& Delta() const { return delta; } - //! Modify the delta. - OutputDataType& Delta() { return delta; } - - //! Get the input width. - size_t InputWidth() const { return inputWidth; } - //! Modify the input width. - size_t& InputWidth() { return inputWidth; } - - //! Get the input height. - size_t InputHeight() const { return inputHeight; } - //! Modify the input height. - size_t& InputHeight() { return inputHeight; } - - //! Get the output width. - size_t OutputWidth() const { return outputWidth; } - //! Modify the output width. - size_t& OutputWidth() { return outputWidth; } - - //! Get the output height. - size_t OutputHeight() const { return outputHeight; } - //! Modify the output height. - size_t& OutputHeight() { return outputHeight; } - - //! Get the input size. - size_t InputSize() const { return inSize; } - - //! Get the output size. - size_t OutputSize() const { return outSize; } - - //! Get the kernel width. - size_t KernelWidth() const { return kernelWidth; } - //! Modify the kernel width. - size_t& KernelWidth() { return kernelWidth; } - - //! Get the kernel height. - size_t KernelHeight() const { return kernelHeight; } - //! Modify the kernel height. - size_t& KernelHeight() { return kernelHeight; } - - //! Get the stride width. - size_t StrideWidth() const { return strideWidth; } - //! Modify the stride width. - size_t& StrideWidth() { return strideWidth; } - - //! Get the stride height. - size_t StrideHeight() const { return strideHeight; } - //! Modify the stride height. - size_t& StrideHeight() { return strideHeight; } - - //! Get the value of the rounding operation. - bool Floor() const { return floor; } - //! Modify the value of the rounding operation. - bool& Floor() { return floor; } - - //! Get the value of the deterministic parameter. - bool Deterministic() const { return deterministic; } - //! Modify the value of the deterministic parameter. - bool& Deterministic() { return deterministic; } - - //! Get the size of the weights. - size_t WeightSize() const { return 0; } - - /** - * Serialize the layer. - */ - template - void serialize(Archive& ar, const uint32_t /* version */); - - private: - /** - * Apply pooling to the input and store the results. - * - * @param input The input to be apply the pooling rule. - * @param output The pooled result. - * @param poolingIndices The pooled indices. - */ - template - void PoolingOperation(const arma::Mat& input, - arma::Mat& output, - arma::Mat& poolingIndices) - { - for (size_t j = 0, colidx = 0; j < output.n_cols; - ++j, colidx += strideHeight) - { - for (size_t i = 0, rowidx = 0; i < output.n_rows; - ++i, rowidx += strideWidth) - { - arma::mat subInput = input( - arma::span(rowidx, rowidx + kernelWidth - 1 - offset), - arma::span(colidx, colidx + kernelHeight - 1 - offset)); - - const size_t idx = pooling.Pooling(subInput); - output(i, j) = subInput(idx); - - if (!deterministic) - { - arma::Mat subIndices = indices(arma::span(rowidx, - rowidx + kernelWidth - 1 - offset), - arma::span(colidx, colidx + kernelHeight - 1 - offset)); - - poolingIndices(i, j) = subIndices(idx); - } - } - } - } - - /** - * Apply unpooling to the input and store the results. - * - * @param error The backward error. - * @param output The pooled result. - * @param poolingIndices The pooled indices. - */ - template - void Unpooling(const arma::Mat& error, - arma::Mat& output, - arma::Mat& poolingIndices) - { - for (size_t i = 0; i < poolingIndices.n_elem; ++i) - { - output(poolingIndices(i)) += error(i); - } - } - - //! Locally-stored width of the pooling window. - size_t kernelWidth; - - //! Locally-stored height of the pooling window. - size_t kernelHeight; - - //! Locally-stored width of the stride operation. - size_t strideWidth; - - //! Locally-stored height of the stride operation. - size_t strideHeight; - - //! Rounding operation used. - bool floor; - - //! Locally-stored number of input channels. - size_t inSize; - - //! Locally-stored number of output channels. - size_t outSize; - - //! Locally-stored reset parameter used to initialize the module once. - bool reset; - - //! Locally-stored input width. - size_t inputWidth; - - //! Locally-stored input height. - size_t inputHeight; - - //! Locally-stored output width. - size_t outputWidth; - - //! Locally-stored output height. - size_t outputHeight; - - //! If true use maximum a posteriori during the forward pass. - bool deterministic; - - //! Locally-stored stored rounding offset. - size_t offset; - - //! Locally-stored number of input units. - size_t batchSize; - - //! Locally-stored output parameter. - arma::cube outputTemp; - - //! Locally-stored transformed input parameter. - arma::cube inputTemp; - - //! Locally-stored transformed output parameter. - arma::cube gTemp; - - //! Locally-stored pooling strategy. - MaxPoolingRule pooling; - - //! Locally-stored delta object. - OutputDataType delta; - - //! Locally-stored gradient object. - OutputDataType gradient; - - //! Locally-stored output parameter object. - OutputDataType outputParameter; - - //! Locally-stored indices matrix parameter. - arma::Mat indices; - - //! Locally-stored indices column parameter. - arma::Col indicesCol; - - //! Locally-stored pooling indicies. - std::vector poolingIndices; -}; // class MaxPooling - -} // namespace ann -} // namespace mlpack - -// Include implementation. -#include "max_pooling_impl.hpp" - -#endif diff --git a/src/mlpack/methods/ann/layer/kmax_pooling_impl.hpp b/src/mlpack/methods/ann/layer/kmax_pooling_impl.hpp deleted file mode 100644 index cbc17904c4..0000000000 --- a/src/mlpack/methods/ann/layer/kmax_pooling_impl.hpp +++ /dev/null @@ -1,165 +0,0 @@ -/** - * @file methods/ann/layer/max_pooling_impl.hpp - * @author Marcus Edel - * @author Nilay Jain - * - * Implementation of the MaxPooling class. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_MAX_POOLING_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_MAX_POOLING_IMPL_HPP - -// In case it hasn't yet been included. -#include "max_pooling.hpp" - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -template -MaxPooling::MaxPooling() -{ - // Nothing to do here. -} - -template -MaxPooling::MaxPooling( - const size_t kernelWidth, - const size_t kernelHeight, - const size_t strideWidth, - const size_t strideHeight, - const bool floor) : - kernelWidth(kernelWidth), - kernelHeight(kernelHeight), - strideWidth(strideWidth), - strideHeight(strideHeight), - floor(floor), - inSize(0), - outSize(0), - reset(false), - inputWidth(0), - inputHeight(0), - outputWidth(0), - outputHeight(0), - deterministic(false), - offset(0), - batchSize(0) -{ - // Nothing to do here. -} - -template -template -void MaxPooling::Forward( - const arma::Mat& input, arma::Mat& output) -{ - batchSize = input.n_cols; - inSize = input.n_elem / (inputWidth * inputHeight * batchSize); - inputTemp = arma::cube(const_cast&>(input).memptr(), - inputWidth, inputHeight, batchSize * inSize, false, false); - - if (floor) - { - outputWidth = std::floor((inputWidth - - (double) kernelWidth) / (double) strideWidth + 1); - outputHeight = std::floor((inputHeight - - (double) kernelHeight) / (double) strideHeight + 1); - offset = 0; - } - else - { - outputWidth = std::ceil((inputWidth - - (double) kernelWidth) / (double) strideWidth + 1); - outputHeight = std::ceil((inputHeight - - (double) kernelHeight) / (double) strideHeight + 1); - offset = 1; - } - - outputTemp = arma::zeros >(outputWidth, outputHeight, - batchSize * inSize); - - if (!deterministic) - { - poolingIndices.push_back(outputTemp); - } - - if (!reset) - { - size_t elements = inputWidth * inputHeight; - indicesCol = arma::linspace >(0, (elements - 1), - elements); - - indices = arma::Mat(indicesCol.memptr(), inputWidth, inputHeight); - - reset = true; - } - - for (size_t s = 0; s < inputTemp.n_slices; s++) - { - if (!deterministic) - { - PoolingOperation(inputTemp.slice(s), outputTemp.slice(s), - poolingIndices.back().slice(s)); - } - else - { - PoolingOperation(inputTemp.slice(s), outputTemp.slice(s), - inputTemp.slice(s)); - } - } - - output = arma::Mat(outputTemp.memptr(), outputTemp.n_elem / batchSize, - batchSize); - - outputWidth = outputTemp.n_rows; - outputHeight = outputTemp.n_cols; - outSize = batchSize * inSize; -} - -template -template -void MaxPooling::Backward( - const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) -{ - arma::cube mappedError = arma::cube(((arma::Mat&) gy).memptr(), - outputWidth, outputHeight, outSize, false, false); - - gTemp = arma::zeros(inputTemp.n_rows, - inputTemp.n_cols, inputTemp.n_slices); - - for (size_t s = 0; s < mappedError.n_slices; s++) - { - Unpooling(mappedError.slice(s), gTemp.slice(s), - poolingIndices.back().slice(s)); - } - - poolingIndices.pop_back(); - - g = arma::mat(gTemp.memptr(), gTemp.n_elem / batchSize, batchSize); -} - -template -template -void MaxPooling::serialize( - Archive& ar, - const uint32_t /* version */) -{ - ar(CEREAL_NVP(kernelWidth)); - ar(CEREAL_NVP(kernelHeight)); - ar(CEREAL_NVP(strideWidth)); - ar(CEREAL_NVP(strideHeight)); - ar(CEREAL_NVP(batchSize)); - ar(CEREAL_NVP(floor)); - ar(CEREAL_NVP(inputWidth)); - ar(CEREAL_NVP(inputHeight)); - ar(CEREAL_NVP(outputWidth)); - ar(CEREAL_NVP(outputHeight)); -} - -} // namespace ann -} // namespace mlpack - -#endif From 43fb21edcc9bf264b2ba634642d8d60b99c3c3c5 Mon Sep 17 00:00:00 2001 From: Abhinav Anand Date: Tue, 9 Feb 2021 09:24:58 +0530 Subject: [PATCH 10/37] Minor code quality changes --- src/mlpack/methods/ann/layer/isrlu.hpp | 3 ++- src/mlpack/methods/ann/layer/isrlu_impl.hpp | 4 ++-- src/mlpack/tests/activation_functions_test.cpp | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/ann/layer/isrlu.hpp b/src/mlpack/methods/ann/layer/isrlu.hpp index d8fad5be47..7d8eda954b 100644 --- a/src/mlpack/methods/ann/layer/isrlu.hpp +++ b/src/mlpack/methods/ann/layer/isrlu.hpp @@ -8,7 +8,8 @@ * * @code * @article{ - * author = {Carlile, Brad and Delamarter, Guy and Kinney, Paul and Marti, Akiko and Whitney, Brian}, + * author = {Carlile, Brad and Delamarter, Guy and Kinney, Paul and Marti, + * Akiko and Whitney, Brian}, * title = {Improving deep learning by inverse square root linear units (ISRLUs)}, * year = {2017}, * url = {https://arxiv.org/pdf/1710.09967.pdf} diff --git a/src/mlpack/methods/ann/layer/isrlu_impl.hpp b/src/mlpack/methods/ann/layer/isrlu_impl.hpp index 8e74241305..651f1f1a28 100644 --- a/src/mlpack/methods/ann/layer/isrlu_impl.hpp +++ b/src/mlpack/methods/ann/layer/isrlu_impl.hpp @@ -10,7 +10,7 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #ifndef MLPACK_METHODS_ANN_LAYER_ISRLU_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_V_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_ISRLU_IMPL_HPP // In case it hasn't yet been included. #include "isrlu.hpp" @@ -42,7 +42,7 @@ void ISRLU::Forward( for (size_t i = 0; i < input.n_elem; ++i) { derivative(i) = (input(i) >= 0) ? 1 : - std:pow(1 / std::sqrt(1 + alpha*input(i)*input(i)), 3); + std::pow(1 / std::sqrt(1 + alpha*input(i)*input(i)), 3); } } } diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index 3ac6437cd0..847051cfac 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -566,7 +566,7 @@ void CheckCELUDerivativeCorrect(const arma::colvec input, * @param target Target data used to evaluate the ISRLU activation. */ void CheckISRLUActivationCorrect(const arma::colvec input, - const arma::colvec target) + const arma::colvec target) { // Initialize ISRLU object with alpha = 1.0. ISRLU<> lrf(1.0); @@ -588,7 +588,7 @@ void CheckISRLUActivationCorrect(const arma::colvec input, * @param target Target data used to evaluate the ISRLU activation. */ void CheckISRLUDerivativeCorrect(const arma::colvec input, - const arma::colvec target) + const arma::colvec target) { // Initialize ISRLU object with alpha = 1.0. ISRLU<> lrf(1.0); From cf9af9204500433a80e676f0bbacf0e7420442f9 Mon Sep 17 00:00:00 2001 From: Abhinav Anand Date: Tue, 9 Feb 2021 10:59:05 +0530 Subject: [PATCH 11/37] changed layer_types --- src/mlpack/methods/ann/layer/layer_types.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 72c3df52ce..718dd8c12e 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -235,7 +235,8 @@ using MoreTypes = boost::variant< VirtualBatchNorm*, RBF*, BaseLayer*, - PositionalEncoding* + PositionalEncoding*, + ISRLU* >; template @@ -273,7 +274,6 @@ using LayerTypes = boost::variant< FlexibleReLU*, GRU*, HardTanH*, - ISRLU*, Join*, LayerNorm*, LeakyReLU*, From 3c3d936f14dff7801bcd68c3eedf870c091b930d Mon Sep 17 00:00:00 2001 From: Abhinav Anand Date: Tue, 9 Feb 2021 11:51:31 +0530 Subject: [PATCH 12/37] fixed test case --- src/mlpack/tests/activation_functions_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index 847051cfac..642aa84dc4 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -1047,9 +1047,9 @@ TEST_CASE("ISRLUFunctionTest", "[ActivationFunctionsTest]") const arma::colvec desiredActivations("-0.89442719 3.2 4.5 \ -0.99995020 1 -0.70710678 2 0"); - const arma::colvec desiredDerivatives("0.74535599 1 1 \ - 0.70712438 1 \ - 0.81649658 1 1"); + const arma::colvec desiredDerivatives("0.41408666 1 1 \ + 0.35357980 1 \ + 0.54433105 1 1"); CheckISRLUActivationCorrect(activationData, desiredActivations); CheckISRLUDerivativeCorrect(desiredActivations, desiredDerivatives); From 1ae6336ab4d49cb56c2f30ae1796542ed5407d99 Mon Sep 17 00:00:00 2001 From: abh2k <41710346+abh2k@users.noreply.github.com> Date: Fri, 12 Feb 2021 05:17:23 +0530 Subject: [PATCH 13/37] Update pixel_shuffle_impl.hpp Updated Cereal_nvp --- .../methods/ann/layer/pixel_shuffle_impl.hpp | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp index 796ee54071..17c7c93980 100644 --- a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp +++ b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp @@ -133,16 +133,17 @@ void PixelShuffle::serialize( Archive& ar, const unsigned int /* version */) { - ar & BOOST_SERIALIZATION_NVP(delta); - ar & BOOST_SERIALIZATION_NVP(outputParameter); - ar & BOOST_SERIALIZATION_NVP(upscaleFactor); - ar & BOOST_SERIALIZATION_NVP(height); - ar & BOOST_SERIALIZATION_NVP(width); - ar & BOOST_SERIALIZATION_NVP(size); - ar & BOOST_SERIALIZATION_NVP(batchSize); - ar & BOOST_SERIALIZATION_NVP(outputHeight); - ar & BOOST_SERIALIZATION_NVP(outputWidth); - ar & BOOST_SERIALIZATION_NVP(sizeOut); + ar(CEREAL_NVP(delta)); + ar(CEREAL_NVP(outputParameter)); + ar(CEREAL_NVP(upscaleFactor)); + ar(CEREAL_NVP(height)); + ar(CEREAL_NVP(width)); + ar(CEREAL_NVP(size)); + ar(CEREAL_NVP(batchSize)); + ar(CEREAL_NVP(outputHeight)); + ar(CEREAL_NVP(outputHeight)); + ar(CEREAL_NVP(outputWidth)); + ar(CEREAL_NVP(sizeOut)); } } // namespace ann From 1aae47ae0b3cc35bb5aff54d9f4a12f4f70a939f Mon Sep 17 00:00:00 2001 From: abh2k <41710346+abh2k@users.noreply.github.com> Date: Fri, 12 Feb 2021 06:36:04 +0530 Subject: [PATCH 14/37] Update layer_types.hpp Added pixel shuffle to more_types --- src/mlpack/methods/ann/layer/layer_types.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 14bac39a81..c1e1987f70 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -220,6 +220,7 @@ class AdaptiveMeanPooling; using MoreTypes = boost::variant< Linear3D*, + PixelShuffle*, Glimpse*, Highway*, MultiheadAttention*, @@ -290,7 +291,6 @@ using LayerTypes = boost::variant< NoisyLinear*, Padding*, PReLU*, - PixelShuffle*, Softmax*, SpatialDropout*, TransposedConvolution, From efcba7760369c835ad44a2a00105840081f0185a Mon Sep 17 00:00:00 2001 From: abh2k <41710346+abh2k@users.noreply.github.com> Date: Sat, 13 Feb 2021 04:31:06 +0530 Subject: [PATCH 15/37] Update isrlu_impl.hpp Use elementwise multiplication while calculating `output(i)` and used `output(i)` while calculating `derivative(i)` --- src/mlpack/methods/ann/layer/isrlu_impl.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/ann/layer/isrlu_impl.hpp b/src/mlpack/methods/ann/layer/isrlu_impl.hpp index 651f1f1a28..b69ffbf8bb 100644 --- a/src/mlpack/methods/ann/layer/isrlu_impl.hpp +++ b/src/mlpack/methods/ann/layer/isrlu_impl.hpp @@ -32,8 +32,8 @@ void ISRLU::Forward( output = arma::ones(arma::size(input)); for (size_t i = 0; i < input.n_elem; ++i) { - output(i) = (input(i) >= 0) ? input(i) : input(i) * - (1 / std::sqrt(1 + alpha*input(i)*input(i))); + output(i) = (input(i) >= 0) ? input(i) : input(i) % + (1 / std::sqrt(1 + alpha * (input(i) % input(i)))); } if (!deterministic) @@ -42,7 +42,7 @@ void ISRLU::Forward( for (size_t i = 0; i < input.n_elem; ++i) { derivative(i) = (input(i) >= 0) ? 1 : - std::pow(1 / std::sqrt(1 + alpha*input(i)*input(i)), 3); + std::pow(output(i), 3) / std::pow(input(i), 3); } } } From 517a3a5e8d1ccfa75fa90e6e4e2239fa0494585b Mon Sep 17 00:00:00 2001 From: Anush V Kini Date: Sun, 14 Feb 2021 12:54:35 +0530 Subject: [PATCH 16/37] Added Hard Swish Function Implementation and Test Skeleton --- .../ann/activation_functions/CMakeLists.txt | 1 + .../hard_swish_function.hpp | 116 ++++++++++++++++++ .../tests/activation_functions_test.cpp | 20 +++ 3 files changed, 137 insertions(+) create mode 100644 src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp diff --git a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt index fd4e765006..d5c0868c1c 100644 --- a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt +++ b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt @@ -19,6 +19,7 @@ set(SOURCES multi_quadratic_function.hpp poisson1_function.hpp gaussian_function.hpp + hard_swish_function.hpp ) # Add directory name to sources. diff --git a/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp b/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp new file mode 100644 index 0000000000..d3526da428 --- /dev/null +++ b/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp @@ -0,0 +1,116 @@ +/** + * @file methods/ann/activation_functions/hard_swish_function.hpp + * @author Anush Kini + * + * Definition and implementation of the Hard Swish function as described by + * Howard A, Sandler M, Chu G, Chen LC, Chen B, Tan M, Wang W, Zhu Y, Pang R, + * Vasudevan V and Le QV. + * For more information, see the following paper. + * + * @code + * @misc{ + * author = {Howard A, Sandler M, Chu G, Chen LC, Chen B, Tan M, Wang W, + * Zhu Y, Pang R, Vasudevan V and Le QV}, + * title = {Searching for MobileNetV3}, + * year = {2019} + * } + * @endcode + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_HARD_SWISH_FUNCTION_HPP +#define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_HARD_SWISH_FUNCTION_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { +/** + * The Hard Swish function, defined by + * + * @f{eqnarray*}{ + * f(x) &=& \begin{cases} + * 0 & x \leq -3\\ + * x & x \geq +3\\ + * \frac{x * (x + 3)}{6} & otherwise\\ + * \end{cases} \\ + * f'(x) &=& \begin{cases} + * 0 & x \leq -3\\ + * 1 & x \geq +3\\ + * \frac{2x + 3}{6} & otherwise\\ + * \end{cases} + * @f} + */ +class HardSwishFunction +{ + public: + /** + * Computes the Hard Swish function. + * + * @param x Input data. + * @return f(x). + */ + static double Fn(const double x) + { + double x2 = x + 3.0; + x2 = x2 > 0.0 ? x2 : 0.0; + x2 = x2 < 6.0 ? x2 : 6.0; + x2 = x * x2 / 6.0; + + return x2; + } + + /** + * Computes the Hard Swish function. + * + * @param x Input data. + * @param y The resulting output activation. + */ + template + static void Fn(const InputVecType &x, OutputVecType &y) + { + y.set_size(size(x)); + + for (size_t i = 0; i < x.n_elem; i++) + y(i) = Fn(x(i)); + } + + /** + * Computes the first derivative of the Hard Swish function. + * + * @param y Input data. + * @return f'(x). + */ + static double Deriv(const double y) + { + if (y <= -3) + return 0; + else if (y >= 3) + return 1; + + return (2*y + 3.0)/6.0; + } + + /** + * Computes the first derivatives of the Hard Swish function. + * + * @param y Input activations. + * @param x The resulting derivatives. + */ + template + static void Deriv(const InputVecType &y, OutputVecType &x) + { + x.set_size(size(y)); + + for (size_t i = 0; i < y.n_elem; i++) + x(i) = Deriv(y(i)); + } +}; // class HardSwishFunction + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index d88c0a5109..f99f9781e8 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -1134,3 +1134,23 @@ TEST_CASE("SoftminFunctionTest", "[ActivationFunctionsTest]") CheckSoftminDerivativeCorrect(activationData, desiredDerivatives); } + +/** + * Basic test of the Hard Swish function. + */ +TEST_CASE("HardSwishFunctionTest", "[ActivationFunctionsTest]") +{ + // Randomly generated data. + const arma::colvec activationData("3.6544 -1.9714 -5.2277 1.5448 2.1164"); + + // Calculated from torch.nn.Hardswish. + const arma::colvec desiredActivations("3.6544 -0.3380 0 1.1701 1.8047"); + + // Hand Calculated Values. + const arma::colvec desiredDerivatives("1 "); + + CheckSoftminActivationCorrect(activationData, + desiredActivations); + CheckSoftminDerivativeCorrect(activationData, + desiredDerivatives); +} From 990803af2b4904279c49cd2374e13f2918be21f9 Mon Sep 17 00:00:00 2001 From: Anush V Kini Date: Sun, 14 Feb 2021 19:31:57 +0530 Subject: [PATCH 17/37] Fixes for failing test --- src/mlpack/methods/ann/layer/base_layer.hpp | 13 +++++++++++++ src/mlpack/tests/activation_functions_test.cpp | 16 +++++++++------- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/mlpack/methods/ann/layer/base_layer.hpp b/src/mlpack/methods/ann/layer/base_layer.hpp index 8429c818a7..ae49f30fe6 100644 --- a/src/mlpack/methods/ann/layer/base_layer.hpp +++ b/src/mlpack/methods/ann/layer/base_layer.hpp @@ -27,6 +27,7 @@ #include #include #include +#include namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -50,6 +51,7 @@ namespace ann /** Artificial Neural Network. */ { * - ELiSHLayer * - ElliotLayer * - GaussianLayer + * - HardSwishLayer * * @tparam ActivationFunction Activation function used for the embedding layer. * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, @@ -277,6 +279,17 @@ template < using GaussianFunctionLayer = BaseLayer< ActivationFunction, InputDataType, OutputDataType>; +/** + * Standard HardSwish-Layer using the HardSwish activation function. + */ +template < + class ActivationFunction = HardSwishFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using HardSwishFunctionLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; + } // namespace ann } // namespace mlpack diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index f99f9781e8..56682880da 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include "catch.hpp" @@ -1143,14 +1144,15 @@ TEST_CASE("HardSwishFunctionTest", "[ActivationFunctionsTest]") // Randomly generated data. const arma::colvec activationData("3.6544 -1.9714 -5.2277 1.5448 2.1164"); - // Calculated from torch.nn.Hardswish. - const arma::colvec desiredActivations("3.6544 -0.3380 0 1.1701 1.8047"); + // Hand Calculated Values. from torch.nn.Hardswish. + const arma::colvec desiredActivations("3.6544 -0.3379636 0.0 \ + 1.1701345 1.8047248"); // Hand Calculated Values. - const arma::colvec desiredDerivatives("1 "); + const arma::colvec desiredDerivatives("1.0 0.38734546 0.5 \ + 0.89004483 1.1015749"); - CheckSoftminActivationCorrect(activationData, - desiredActivations); - CheckSoftminDerivativeCorrect(activationData, - desiredDerivatives); + CheckActivationCorrect(activationData, desiredActivations); + CheckDerivativeCorrect + (desiredActivations, desiredDerivatives); } From 665c750e1f56a12cf014eac180b1862a9b9efaa1 Mon Sep 17 00:00:00 2001 From: Anush V Kini Date: Sun, 14 Feb 2021 22:48:25 +0530 Subject: [PATCH 18/37] Some more comment fixes --- .../methods/ann/activation_functions/hard_swish_function.hpp | 2 +- src/mlpack/tests/activation_functions_test.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp b/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp index d3526da428..f2780aac6e 100644 --- a/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp @@ -97,7 +97,7 @@ class HardSwishFunction /** * Computes the first derivatives of the Hard Swish function. * - * @param y Input activations. + * @param y Input data. * @param x The resulting derivatives. */ template diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index 56682880da..20631566e9 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -1144,7 +1144,7 @@ TEST_CASE("HardSwishFunctionTest", "[ActivationFunctionsTest]") // Randomly generated data. const arma::colvec activationData("3.6544 -1.9714 -5.2277 1.5448 2.1164"); - // Hand Calculated Values. from torch.nn.Hardswish. + // Hand Calculated Values. const arma::colvec desiredActivations("3.6544 -0.3379636 0.0 \ 1.1701345 1.8047248"); From 7b70a820f8511d58ecfabcbcca71b14732ca585b Mon Sep 17 00:00:00 2001 From: abh2k <41710346+abh2k@users.noreply.github.com> Date: Mon, 15 Feb 2021 02:37:02 +0530 Subject: [PATCH 19/37] Code style fix Co-authored-by: kartikdutt18 <39593019+kartikdutt18@users.noreply.github.com> --- src/mlpack/methods/ann/layer/pixel_shuffle.hpp | 1 + src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp | 13 ++++--------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle.hpp index c62b0f0ecb..48d89c2fe2 100644 --- a/src/mlpack/methods/ann/layer/pixel_shuffle.hpp +++ b/src/mlpack/methods/ann/layer/pixel_shuffle.hpp @@ -50,6 +50,7 @@ class PixelShuffle public: //! Create the PixelShuffle object. PixelShuffle(); + /** * Create the PixelShuffle object using the specified parameters. * The number of input channels should be an integral multiple of the square diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp index 17c7c93980..9edb09c2eb 100644 --- a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp +++ b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp @@ -20,15 +20,7 @@ namespace ann /** Artificial Neural Network. */ { template PixelShuffle::PixelShuffle() : - upscaleFactor(0), - height(0), - width(0), - size(0), - batchSize(0), - outputHeight(0), - outputWidth(0), - sizeOut(0), - reset(false) + PixelShuffle(0, 0, 0, 0) { // Nothing to do here. } @@ -65,6 +57,7 @@ void PixelShuffle::Forward( outputWidth = width * upscaleFactor; reset = true; } + output.zeros(outputHeight * outputWidth * sizeOut, batchSize); for (size_t n = 0; n < batchSize; n++) { @@ -90,6 +83,7 @@ void PixelShuffle::Forward( } } } + output.col(n) = outputImage; } } @@ -123,6 +117,7 @@ void PixelShuffle::Backward( } } } + g.col(n) = gImage; } } From 3856cdd00ca20995a06d0f16a20935fde538b60e Mon Sep 17 00:00:00 2001 From: Anush V Kini Date: Mon, 15 Feb 2021 10:03:06 +0530 Subject: [PATCH 20/37] Review fixes --- .../methods/ann/activation_functions/hard_swish_function.hpp | 2 +- src/mlpack/tests/activation_functions_test.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp b/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp index f2780aac6e..d308358f31 100644 --- a/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp @@ -91,7 +91,7 @@ class HardSwishFunction else if (y >= 3) return 1; - return (2*y + 3.0)/6.0; + return (2 * y + 3.0) / 6.0; } /** diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index 20631566e9..5e4c5aa217 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -1144,11 +1144,11 @@ TEST_CASE("HardSwishFunctionTest", "[ActivationFunctionsTest]") // Randomly generated data. const arma::colvec activationData("3.6544 -1.9714 -5.2277 1.5448 2.1164"); - // Hand Calculated Values. + // Hand-calculated values. const arma::colvec desiredActivations("3.6544 -0.3379636 0.0 \ 1.1701345 1.8047248"); - // Hand Calculated Values. + // Hand-calculated values. const arma::colvec desiredDerivatives("1.0 0.38734546 0.5 \ 0.89004483 1.1015749"); From 0fe77567948df2aaa1ce711003c2fcdefc49d8cc Mon Sep 17 00:00:00 2001 From: abh2k <41710346+abh2k@users.noreply.github.com> Date: Mon, 15 Feb 2021 10:24:01 +0530 Subject: [PATCH 21/37] Update isrlu_impl.hpp --- src/mlpack/methods/ann/layer/isrlu_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/layer/isrlu_impl.hpp b/src/mlpack/methods/ann/layer/isrlu_impl.hpp index b69ffbf8bb..8ee468849f 100644 --- a/src/mlpack/methods/ann/layer/isrlu_impl.hpp +++ b/src/mlpack/methods/ann/layer/isrlu_impl.hpp @@ -33,7 +33,7 @@ void ISRLU::Forward( for (size_t i = 0; i < input.n_elem; ++i) { output(i) = (input(i) >= 0) ? input(i) : input(i) % - (1 / std::sqrt(1 + alpha * (input(i) % input(i)))); + (1 / arma::sqrt(1 + alpha * (input(i) % input(i)))); } if (!deterministic) @@ -42,7 +42,7 @@ void ISRLU::Forward( for (size_t i = 0; i < input.n_elem; ++i) { derivative(i) = (input(i) >= 0) ? 1 : - std::pow(output(i), 3) / std::pow(input(i), 3); + arma::pow(output(i), 3) / arma::pow(input(i), 3); } } } From 48deeeb3a6b35cf6c5b14913bc3b2bf915eb834c Mon Sep 17 00:00:00 2001 From: abh2k <41710346+abh2k@users.noreply.github.com> Date: Tue, 16 Feb 2021 04:36:30 +0530 Subject: [PATCH 22/37] Changes % to * --- src/mlpack/methods/ann/layer/isrlu_impl.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/ann/layer/isrlu_impl.hpp b/src/mlpack/methods/ann/layer/isrlu_impl.hpp index 8ee468849f..5d73ce0710 100644 --- a/src/mlpack/methods/ann/layer/isrlu_impl.hpp +++ b/src/mlpack/methods/ann/layer/isrlu_impl.hpp @@ -32,8 +32,8 @@ void ISRLU::Forward( output = arma::ones(arma::size(input)); for (size_t i = 0; i < input.n_elem; ++i) { - output(i) = (input(i) >= 0) ? input(i) : input(i) % - (1 / arma::sqrt(1 + alpha * (input(i) % input(i)))); + output(i) = (input(i) >= 0) ? input(i) : input(i) * + (1 / std::sqrt(1 + alpha * (input(i) * input(i)))); } if (!deterministic) @@ -42,7 +42,7 @@ void ISRLU::Forward( for (size_t i = 0; i < input.n_elem; ++i) { derivative(i) = (input(i) >= 0) ? 1 : - arma::pow(output(i), 3) / arma::pow(input(i), 3); + std::pow(output(i), 3) / std::pow(input(i), 3); } } } From f9fbf4d9b51151f9e92cf892eba2bd483c114ebe Mon Sep 17 00:00:00 2001 From: abh2k <41710346+abh2k@users.noreply.github.com> Date: Wed, 17 Feb 2021 03:21:43 +0530 Subject: [PATCH 23/37] Removed space. Co-authored-by: Marcus Edel --- src/mlpack/methods/ann/layer/isrlu_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/layer/isrlu_impl.hpp b/src/mlpack/methods/ann/layer/isrlu_impl.hpp index 5d73ce0710..5e08aaa707 100644 --- a/src/mlpack/methods/ann/layer/isrlu_impl.hpp +++ b/src/mlpack/methods/ann/layer/isrlu_impl.hpp @@ -33,7 +33,7 @@ void ISRLU::Forward( for (size_t i = 0; i < input.n_elem; ++i) { output(i) = (input(i) >= 0) ? input(i) : input(i) * - (1 / std::sqrt(1 + alpha * (input(i) * input(i)))); + (1 / std::sqrt(1 + alpha * (input(i) * input(i)))); } if (!deterministic) From 1994c4fc419e0623938039389fba90e571e56940 Mon Sep 17 00:00:00 2001 From: Anush V Kini Date: Thu, 18 Feb 2021 11:15:15 +0530 Subject: [PATCH 24/37] Fn implementation changed to if else statements --- .../ann/activation_functions/hard_swish_function.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp b/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp index d308358f31..d387e86474 100644 --- a/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp @@ -55,12 +55,12 @@ class HardSwishFunction */ static double Fn(const double x) { - double x2 = x + 3.0; - x2 = x2 > 0.0 ? x2 : 0.0; - x2 = x2 < 6.0 ? x2 : 6.0; - x2 = x * x2 / 6.0; + if (x <= -3) + return 0; + else if (x >= 3) + return x; - return x2; + return x * (x + 3) / 6; } /** From f6b626d7178716d2f414647586aabddf223fb43e Mon Sep 17 00:00:00 2001 From: abh2k <41710346+abh2k@users.noreply.github.com> Date: Fri, 19 Feb 2021 03:13:49 +0530 Subject: [PATCH 25/37] Moved derivative to backward --- src/mlpack/methods/ann/layer/isrlu_impl.hpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/mlpack/methods/ann/layer/isrlu_impl.hpp b/src/mlpack/methods/ann/layer/isrlu_impl.hpp index 5e08aaa707..fc022929ad 100644 --- a/src/mlpack/methods/ann/layer/isrlu_impl.hpp +++ b/src/mlpack/methods/ann/layer/isrlu_impl.hpp @@ -35,16 +35,6 @@ void ISRLU::Forward( output(i) = (input(i) >= 0) ? input(i) : input(i) * (1 / std::sqrt(1 + alpha * (input(i) * input(i)))); } - - if (!deterministic) - { - derivative.set_size(arma::size(input)); - for (size_t i = 0; i < input.n_elem; ++i) - { - derivative(i) = (input(i) >= 0) ? 1 : - std::pow(output(i), 3) / std::pow(input(i), 3); - } - } } template @@ -52,6 +42,15 @@ template void ISRLU::Backward( const DataType& /* input */, const DataType& gy, DataType& g) { + if (!deterministic) + { + derivative.set_size(arma::size(input)); + for (size_t i = 0; i < input.n_elem; ++i) + { + derivative(i) = (input(i) >= 0) ? 1 : + std::pow(1 / std::sqrt(1 + alpha*input(i)*input(i)), 3); + } + } g = gy % derivative; } From 4394e99151cb05d4d45a0e5fb507396035c5d3fb Mon Sep 17 00:00:00 2001 From: abh2k <41710346+abh2k@users.noreply.github.com> Date: Fri, 19 Feb 2021 03:15:22 +0530 Subject: [PATCH 26/37] minor change --- src/mlpack/methods/ann/layer/isrlu_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/layer/isrlu_impl.hpp b/src/mlpack/methods/ann/layer/isrlu_impl.hpp index fc022929ad..57f196b3fe 100644 --- a/src/mlpack/methods/ann/layer/isrlu_impl.hpp +++ b/src/mlpack/methods/ann/layer/isrlu_impl.hpp @@ -40,7 +40,7 @@ void ISRLU::Forward( template template void ISRLU::Backward( - const DataType& /* input */, const DataType& gy, DataType& g) + const DataType& input, const DataType& gy, DataType& g) { if (!deterministic) { From dfd5f142d80480d4c19f088e832689cca3c8f0eb Mon Sep 17 00:00:00 2001 From: abh2k <41710346+abh2k@users.noreply.github.com> Date: Sun, 21 Feb 2021 20:37:58 +0530 Subject: [PATCH 27/37] fixed test case --- src/mlpack/tests/activation_functions_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index 642aa84dc4..2a64068891 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -1052,7 +1052,7 @@ TEST_CASE("ISRLUFunctionTest", "[ActivationFunctionsTest]") 0.54433105 1 1"); CheckISRLUActivationCorrect(activationData, desiredActivations); - CheckISRLUDerivativeCorrect(desiredActivations, desiredDerivatives); + CheckISRLUDerivativeCorrect(activationData, desiredDerivatives); } /** From fd201b7d5e31e538af573a35731c98bff74f7948 Mon Sep 17 00:00:00 2001 From: Abhinav Anand Date: Mon, 22 Feb 2021 06:38:37 +0530 Subject: [PATCH 28/37] removed deterministic parameter and improved co --- src/mlpack/methods/ann/layer/isrlu.hpp | 8 -------- src/mlpack/methods/ann/layer/isrlu_impl.hpp | 14 +++++--------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/mlpack/methods/ann/layer/isrlu.hpp b/src/mlpack/methods/ann/layer/isrlu.hpp index 7d8eda954b..b0a786c6ba 100644 --- a/src/mlpack/methods/ann/layer/isrlu.hpp +++ b/src/mlpack/methods/ann/layer/isrlu.hpp @@ -47,7 +47,6 @@ namespace ann /** Artificial Neural Network. */ { * \right. * @f} * - * In the deterministic mode, there is no computation of the derivative. * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). @@ -106,11 +105,6 @@ class ISRLU //! Modify the non zero gradient. double& Alpha() { return alpha; } - //! Get the value of deterministic parameter. - bool Deterministic() const { return deterministic; } - //! Modify the value of deterministic parameter. - bool& Deterministic() { return deterministic; } - //! Get size of weights. size_t WeightSize() { return 0; } @@ -133,8 +127,6 @@ class ISRLU //! ISRLU Hyperparameter (alpha > 0). double alpha; - //! If true the derivative computation is disabled, see notes above. - bool deterministic; }; // class ISRLU } // namespace ann diff --git a/src/mlpack/methods/ann/layer/isrlu_impl.hpp b/src/mlpack/methods/ann/layer/isrlu_impl.hpp index 57f196b3fe..a91830f51d 100644 --- a/src/mlpack/methods/ann/layer/isrlu_impl.hpp +++ b/src/mlpack/methods/ann/layer/isrlu_impl.hpp @@ -20,8 +20,7 @@ namespace ann /** Artificial Neural Network. */ { template ISRLU::ISRLU(const double alpha) : - alpha(alpha), - deterministic(false) + alpha(alpha) {} template @@ -42,14 +41,11 @@ template void ISRLU::Backward( const DataType& input, const DataType& gy, DataType& g) { - if (!deterministic) + derivative.set_size(arma::size(input)); + for (size_t i = 0; i < input.n_elem; ++i) { - derivative.set_size(arma::size(input)); - for (size_t i = 0; i < input.n_elem; ++i) - { - derivative(i) = (input(i) >= 0) ? 1 : - std::pow(1 / std::sqrt(1 + alpha*input(i)*input(i)), 3); - } + derivative(i) = (input(i) >= 0) ? 1 : + std::pow(1 / std::sqrt(1 + alpha * input(i) * input(i)), 3); } g = gy % derivative; } From 0180ca4f09b82aa875cfc82a59205d492ddb0d4f Mon Sep 17 00:00:00 2001 From: abh2k <41710346+abh2k@users.noreply.github.com> Date: Tue, 23 Feb 2021 12:38:19 +0530 Subject: [PATCH 29/37] Changed author name format --- src/mlpack/methods/ann/layer/isrlu.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/layer/isrlu.hpp b/src/mlpack/methods/ann/layer/isrlu.hpp index b0a786c6ba..fb4cd65947 100644 --- a/src/mlpack/methods/ann/layer/isrlu.hpp +++ b/src/mlpack/methods/ann/layer/isrlu.hpp @@ -8,8 +8,8 @@ * * @code * @article{ - * author = {Carlile, Brad and Delamarter, Guy and Kinney, Paul and Marti, - * Akiko and Whitney, Brian}, + * author = {Carlile, Brad, Delamarter, Guy, Kinney, Paul, Marti, + * Akiko, Whitney, Brian}, * title = {Improving deep learning by inverse square root linear units (ISRLUs)}, * year = {2017}, * url = {https://arxiv.org/pdf/1710.09967.pdf} From 5e695f99c8ea389fcf9b08080c51ce8b0ecc5ad9 Mon Sep 17 00:00:00 2001 From: abh2k <41710346+abh2k@users.noreply.github.com> Date: Tue, 23 Feb 2021 13:55:23 +0530 Subject: [PATCH 30/37] Removed copying --- .../methods/ann/layer/pixel_shuffle_impl.hpp | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp index 9edb09c2eb..89ae8b79d5 100644 --- a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp +++ b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp @@ -61,12 +61,10 @@ void PixelShuffle::Forward( output.zeros(outputHeight * outputWidth * sizeOut, batchSize); for (size_t n = 0; n < batchSize; n++) { - arma::mat inputImage = input.col(n); - arma::mat outputImage = output.col(n); - arma::cube inputTemp(const_cast(inputImage).memptr(), height, - width, size, false, false); - arma::cube outputTemp(const_cast(outputImage).memptr(), - outputHeight, outputWidth, sizeOut, false, false); + arma::cube inputTemp(const_cast(input).memptr(), height, + width, size * batchSize, false, false); + arma::cube outputTemp(const_cast(output).memptr(), + outputHeight, outputWidth, sizeOut * batchSize, false, false); for (size_t c = 0; c < sizeOut ; c++) { @@ -78,13 +76,12 @@ void PixelShuffle::Forward( size_t width_index = w / upscaleFactor; size_t channel_index = (upscaleFactor * (h % upscaleFactor)) + (w % upscaleFactor) + (c * std::pow(upscaleFactor, 2)); - outputTemp(w, h, c) = inputTemp(width_index, height_index, - channel_index); + outputTemp(w, h, c + n * sizeOut) = inputTemp(width_index, height_index, + channel_index + n * size); } } } - output.col(n) = outputImage; } } From 4eff650f9d90b236139117b4c6a69acfbb8f0329 Mon Sep 17 00:00:00 2001 From: abh2k <41710346+abh2k@users.noreply.github.com> Date: Tue, 23 Feb 2021 15:16:44 +0530 Subject: [PATCH 31/37] Remove copying in backward function. --- .../methods/ann/layer/pixel_shuffle_impl.hpp | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp index 89ae8b79d5..7bcf985f45 100644 --- a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp +++ b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp @@ -66,7 +66,7 @@ void PixelShuffle::Forward( arma::cube outputTemp(const_cast(output).memptr(), outputHeight, outputWidth, sizeOut * batchSize, false, false); - for (size_t c = 0; c < sizeOut ; c++) + for (size_t c = 0; c < sizeOut; c++) { for (size_t h = 0; h < outputHeight; h++) { @@ -93,14 +93,12 @@ void PixelShuffle::Backward( g.zeros(arma::size(input)); for (size_t n = 0; n < batchSize; n++) { - arma::mat gyImage = gy.col(n); - arma::mat gImage = g.col(n); - arma::cube gyTemp(const_cast(gyImage).memptr(), outputHeight, - outputWidth, sizeOut, false, false); - arma::cube gTemp(const_cast(gImage).memptr(), height, width, - size, false, false); + arma::cube gyTemp(const_cast(gy).memptr(), outputHeight, + outputWidth, sizeOut * batchSize, false, false); + arma::cube gTemp(const_cast(g).memptr(), height, width, + size * batchSize, false, false); - for (size_t c = 0; c < sizeOut ; c++) + for (size_t c = 0; c < sizeOut; c++) { for (size_t h = 0; h < outputHeight; h++) { @@ -110,12 +108,12 @@ void PixelShuffle::Backward( size_t width_index = w / upscaleFactor; size_t channel_index = (upscaleFactor * (h % upscaleFactor)) + (w % upscaleFactor) + (c * std::pow(upscaleFactor, 2)); - gTemp(width_index, height_index, channel_index) = gyTemp(w, h, c); + gTemp(width_index, height_index, channel_index + n * sizeOut) = gyTemp(w, h, + c + n * size); } } } - g.col(n) = gImage; } } From 4f9eb2845c7d2f94886ead42119e2be2c97f8561 Mon Sep 17 00:00:00 2001 From: abh2k <41710346+abh2k@users.noreply.github.com> Date: Tue, 23 Feb 2021 16:47:45 +0530 Subject: [PATCH 32/37] Update pixel_shuffle_impl.hpp --- src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp index 7bcf985f45..c062c2a029 100644 --- a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp +++ b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp @@ -76,8 +76,8 @@ void PixelShuffle::Forward( size_t width_index = w / upscaleFactor; size_t channel_index = (upscaleFactor * (h % upscaleFactor)) + (w % upscaleFactor) + (c * std::pow(upscaleFactor, 2)); - outputTemp(w, h, c + n * sizeOut) = inputTemp(width_index, height_index, - channel_index + n * size); + outputTemp(w, h, c + n * size) = inputTemp(width_index, height_index, + channel_index + n * sizeOut); } } } From cd167d98ad75c45bb9ef72589807450a51ce57a4 Mon Sep 17 00:00:00 2001 From: abh2k <41710346+abh2k@users.noreply.github.com> Date: Tue, 23 Feb 2021 18:13:32 +0530 Subject: [PATCH 33/37] Update pixel_shuffle_impl.hpp --- src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp index c062c2a029..7aa976e4d6 100644 --- a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp +++ b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp @@ -76,8 +76,8 @@ void PixelShuffle::Forward( size_t width_index = w / upscaleFactor; size_t channel_index = (upscaleFactor * (h % upscaleFactor)) + (w % upscaleFactor) + (c * std::pow(upscaleFactor, 2)); - outputTemp(w, h, c + n * size) = inputTemp(width_index, height_index, - channel_index + n * sizeOut); + outputTemp(w, h, c + n * sizeOut) = inputTemp(width_index, height_index, + channel_index + n * size); } } } @@ -108,8 +108,8 @@ void PixelShuffle::Backward( size_t width_index = w / upscaleFactor; size_t channel_index = (upscaleFactor * (h % upscaleFactor)) + (w % upscaleFactor) + (c * std::pow(upscaleFactor, 2)); - gTemp(width_index, height_index, channel_index + n * sizeOut) = gyTemp(w, h, - c + n * size); + gTemp(width_index, height_index, channel_index + n * size) = gyTemp(w, h, + c + n * sizeOut); } } } From b1cb6255e68328164f0452a70c5b45ab648225a9 Mon Sep 17 00:00:00 2001 From: abh2k <41710346+abh2k@users.noreply.github.com> Date: Wed, 24 Feb 2021 11:00:31 +0530 Subject: [PATCH 34/37] changed authors name Co-authored-by: Ryan Birmingham --- src/mlpack/methods/ann/layer/isrlu.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/layer/isrlu.hpp b/src/mlpack/methods/ann/layer/isrlu.hpp index fb4cd65947..b0a786c6ba 100644 --- a/src/mlpack/methods/ann/layer/isrlu.hpp +++ b/src/mlpack/methods/ann/layer/isrlu.hpp @@ -8,8 +8,8 @@ * * @code * @article{ - * author = {Carlile, Brad, Delamarter, Guy, Kinney, Paul, Marti, - * Akiko, Whitney, Brian}, + * author = {Carlile, Brad and Delamarter, Guy and Kinney, Paul and Marti, + * Akiko and Whitney, Brian}, * title = {Improving deep learning by inverse square root linear units (ISRLUs)}, * year = {2017}, * url = {https://arxiv.org/pdf/1710.09967.pdf} From 95e756dbb4f5e400dd58e2d7ba71a1088915be14 Mon Sep 17 00:00:00 2001 From: abh2k <41710346+abh2k@users.noreply.github.com> Date: Thu, 25 Feb 2021 18:15:22 +0530 Subject: [PATCH 35/37] Adding co-author --- src/mlpack/methods/ann/layer/pixel_shuffle.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle.hpp index 48d89c2fe2..f425d7c508 100644 --- a/src/mlpack/methods/ann/layer/pixel_shuffle.hpp +++ b/src/mlpack/methods/ann/layer/pixel_shuffle.hpp @@ -1,6 +1,7 @@ /** * @file methods/ann/layer/pixel_shuffle.hpp * @author Anjishnu Mukherjee + * @author Abhinav Anand * * Definition of the PixelShuffle class. * From 6e040a48aebb4ec213e069473e81798c1a418a09 Mon Sep 17 00:00:00 2001 From: abh2k <41710346+abh2k@users.noreply.github.com> Date: Thu, 25 Feb 2021 18:16:42 +0530 Subject: [PATCH 36/37] Adding co-author --- src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp index 7aa976e4d6..f56f708981 100644 --- a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp +++ b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp @@ -1,6 +1,7 @@ /** * @file methods/ann/layer/pixel_shuffle_impl.hpp * @author Anjishnu Mukherjee + * @author Abhinav Anand * * Implementation of the PixelShuffle class. * From cdede9018bdf5f22735b11b3815c9a01662ea03d Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sat, 27 Feb 2021 15:59:50 -0500 Subject: [PATCH 37/37] Make sure feedforward_network_2_test.cpp gets compiled and run. --- src/mlpack/tests/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index e6dd629266..e94c73e0d3 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -34,6 +34,7 @@ add_executable(mlpack_test facilities_test.cpp fastmks_test.cpp feedforward_network_test.cpp + feedforward_network_2_test.cpp gan_test.cpp gmm_test.cpp hmm_test.cpp