diff --git a/HISTORY.md b/HISTORY.md index 1039ed9d13..12692b8308 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,7 @@ ### mlpack ?.?.? ###### ????-??-?? + * Added Pixel Shuffle layer (#2563). + * Add "check_input_matrices" option to python bindings that checks for NaN and inf values in all the input matrices (#2787). @@ -52,7 +54,6 @@ ### mlpack 3.4.0 ###### 2020-09-01 - * Issue warnings when metrics produce NaNs in KFoldCV (#2595). * Added bindings for _R_ during Google Summer of Code (#2556). diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt index d96d64079a..2b181012c7 100644 --- a/src/mlpack/methods/ann/layer/CMakeLists.txt +++ b/src/mlpack/methods/ann/layer/CMakeLists.txt @@ -85,6 +85,8 @@ set(SOURCES noisylinear_impl.hpp parametric_relu.hpp parametric_relu_impl.hpp + pixel_shuffle.hpp + pixel_shuffle_impl.hpp positional_encoding.hpp positional_encoding_impl.hpp recurrent.hpp diff --git a/src/mlpack/methods/ann/layer/layer.hpp b/src/mlpack/methods/ann/layer/layer.hpp index 9cf806b7e4..6d13a26772 100644 --- a/src/mlpack/methods/ann/layer/layer.hpp +++ b/src/mlpack/methods/ann/layer/layer.hpp @@ -58,6 +58,7 @@ #include "noisylinear.hpp" #include "padding.hpp" #include "parametric_relu.hpp" +#include "pixel_shuffle.hpp" #include "positional_encoding.hpp" #include "recurrent_attention.hpp" #include "recurrent.hpp" diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 77244879da..2532efecfe 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -222,6 +223,7 @@ class AdaptiveMeanPooling; using MoreTypes = boost::variant< Linear3D*, LpPooling*, + PixelShuffle*, Glimpse*, Highway*, MultiheadAttention*, diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle.hpp new file mode 100644 index 0000000000..f425d7c508 --- /dev/null +++ b/src/mlpack/methods/ann/layer/pixel_shuffle.hpp @@ -0,0 +1,181 @@ +/** + * @file methods/ann/layer/pixel_shuffle.hpp + * @author Anjishnu Mukherjee + * @author Abhinav Anand + * + * Definition of the PixelShuffle class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_HPP +#define MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * Implementation of the PixelShuffle layer. + * + * For more information, refer to the following paper, + * + * @code + * @article{Shi16, + * author = {Wenzhe Shi, Jose Caballero,Ferenc Huszár, Johannes Totz, + * Andrew P. Aitken, Rob Bishop, Daniel Rueckert, Zehan Wang}, + * title = {Real-Time Single Image and Video Super-Resolution Using an + * Efficient Sub-Pixel Convolutional Neural Network}, + * journal = {CoRR}, + * volume = {abs/1609.05158}, + * year = {2016}, + * url = {https://arxiv.org/abs/1609.05158}, + * eprint = {1609.05158}, + * } + * @endcode + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class PixelShuffle +{ + public: + //! Create the PixelShuffle object. + PixelShuffle(); + + /** + * Create the PixelShuffle object using the specified parameters. + * The number of input channels should be an integral multiple of the square + * of the upscale factor. + * + * @param upscaleFactor The scaling factor for Pixel Shuffle. + * @param height The height of each input image. + * @param width The width of each input image. + * @param size The number of channels of each input image. + */ + PixelShuffle(const size_t upscaleFactor, + const size_t height, + const size_t width, + const size_t size); + + /** + * Ordinary feed forward pass of the PixelShuffle layer. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + template + void Forward(const arma::Mat& input, arma::Mat& output); + + /** + * Ordinary feed backward pass of the PixelShuffle layer. + * + * @param input The propagated input activation. + * @param gy The backpropagated error. + * @param g The calculated gradient. + */ + template + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the upscale factor. + size_t UpscaleFactor() const { return upscaleFactor; } + + //! Modify the upscale factor. + size_t& UpscaleFactor() { return upscaleFactor; } + + //! Get the input image height. + size_t InputHeight() const { return height; } + + //! Modify the input image height. + size_t& InputHeight() { return height; } + + //! Get the input image width. + size_t InputWidth() const { return width; } + + //! Modify the input image width. + size_t& InputWidth() { return width; } + + //! Get the number of input channels. + size_t InputChannels() const { return size; } + + //! Modify the number of input channels. + size_t& InputChannels() { return size; } + + //! Get the output image height. + size_t OutputHeight() const { return outputHeight; } + + //! Get the output image width. + size_t OutputWidth() const { return outputWidth; } + + //! Get the number of output channels. + size_t OutputChannels() const { return sizeOut; } + + /** + * Serialize the layer. + */ + template + void serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! The scaling factor for Pixel Shuffle. + size_t upscaleFactor; + + //! The height of each input image. + size_t height; + + //! The width of each input image. + size_t width; + + //! The number of channels of each input image. + size_t size; + + //! The number of images in the batch. + size_t batchSize; + + //! The height of each output image. + size_t outputHeight; + + //! The width of each output image. + size_t outputWidth; + + //! The number of channels of each output image. + size_t sizeOut; + + //! A boolean used to do some internal calculations once initially. + bool reset; +}; // class PixelShuffle + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "pixel_shuffle_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp new file mode 100644 index 0000000000..f56f708981 --- /dev/null +++ b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp @@ -0,0 +1,143 @@ +/** + * @file methods/ann/layer/pixel_shuffle_impl.hpp + * @author Anjishnu Mukherjee + * @author Abhinav Anand + * + * Implementation of the PixelShuffle class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_IMPL_HPP + +// In case it hasn't yet been included. +#include "pixel_shuffle.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +PixelShuffle::PixelShuffle() : + PixelShuffle(0, 0, 0, 0) +{ + // Nothing to do here. +} + +template +PixelShuffle::PixelShuffle( + const size_t upscaleFactor, + const size_t height, + const size_t width, + const size_t size) : + upscaleFactor(upscaleFactor), + height(height), + width(width), + size(size), + batchSize(0), + outputHeight(0), + outputWidth(0), + sizeOut(0), + reset(false) +{ + // Nothing to do here. +} + +template +template +void PixelShuffle::Forward( + const arma::Mat& input, arma::Mat& output) +{ + if (!reset) + { + batchSize = input.n_cols; + sizeOut = size / std::pow(upscaleFactor, 2); + outputHeight = height * upscaleFactor; + outputWidth = width * upscaleFactor; + reset = true; + } + + output.zeros(outputHeight * outputWidth * sizeOut, batchSize); + for (size_t n = 0; n < batchSize; n++) + { + arma::cube inputTemp(const_cast(input).memptr(), height, + width, size * batchSize, false, false); + arma::cube outputTemp(const_cast(output).memptr(), + outputHeight, outputWidth, sizeOut * batchSize, false, false); + + for (size_t c = 0; c < sizeOut; c++) + { + for (size_t h = 0; h < outputHeight; h++) + { + for (size_t w = 0; w < outputWidth; w++) + { + size_t height_index = h / upscaleFactor; + size_t width_index = w / upscaleFactor; + size_t channel_index = (upscaleFactor * (h % upscaleFactor)) + + (w % upscaleFactor) + (c * std::pow(upscaleFactor, 2)); + outputTemp(w, h, c + n * sizeOut) = inputTemp(width_index, height_index, + channel_index + n * size); + } + } + } + + } +} + +template +template +void PixelShuffle::Backward( + const arma::Mat& input, const arma::Mat& gy, arma::Mat& g) +{ + g.zeros(arma::size(input)); + for (size_t n = 0; n < batchSize; n++) + { + arma::cube gyTemp(const_cast(gy).memptr(), outputHeight, + outputWidth, sizeOut * batchSize, false, false); + arma::cube gTemp(const_cast(g).memptr(), height, width, + size * batchSize, false, false); + + for (size_t c = 0; c < sizeOut; c++) + { + for (size_t h = 0; h < outputHeight; h++) + { + for (size_t w = 0; w < outputWidth; w++) + { + size_t height_index = h / upscaleFactor; + size_t width_index = w / upscaleFactor; + size_t channel_index = (upscaleFactor * (h % upscaleFactor)) + + (w % upscaleFactor) + (c * std::pow(upscaleFactor, 2)); + gTemp(width_index, height_index, channel_index + n * size) = gyTemp(w, h, + c + n * sizeOut); + } + } + } + + } +} + +template +template +void PixelShuffle::serialize( + Archive& ar, + const unsigned int /* version */) +{ + ar(CEREAL_NVP(delta)); + ar(CEREAL_NVP(outputParameter)); + ar(CEREAL_NVP(upscaleFactor)); + ar(CEREAL_NVP(height)); + ar(CEREAL_NVP(width)); + ar(CEREAL_NVP(size)); + ar(CEREAL_NVP(batchSize)); + ar(CEREAL_NVP(outputHeight)); + ar(CEREAL_NVP(outputHeight)); + ar(CEREAL_NVP(outputWidth)); + ar(CEREAL_NVP(sizeOut)); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index dbb5266798..54105b1f9e 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -4606,6 +4606,106 @@ TEST_CASE("TransposedConvolutionWeightInitializationTest", "[ANNLayerTest]") } /** + * Simple Test for PixelShuffle layer. + */ +TEST_CASE("PixelShuffleLayerTest", "[ANNLayerTest]") +{ + arma::mat input1, output1, gy1, g1, outputExpected1, gExpected1; + arma::mat input2, output2, gy2, g2, outputExpected2, gExpected2; + PixelShuffle<> module1(2, 2, 2, 4); + PixelShuffle<> module2(2, 2, 2, 4); + + // Input is a single image, of size (2,2) and having 4 channels. + input1 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 + << 0 << 0 << arma::endr; + gy1 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8 + << 12 << 16 << arma::endr; + + // Calculated using torch.nn.PixelShuffle(). + outputExpected1 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0 + << 0 << 0 << 0 << 0 << arma::endr; + gExpected1 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12 + << 6 << 14 << 8 << 16 << arma::endr; + + input1 = input1.t(); + outputExpected1 = outputExpected1.t(); + gy1 = gy1.t(); + gExpected1 = gExpected1.t(); + + // Check the Forward pass of the layer. + module1.Forward(input1, output1); + CheckMatrices(output1, outputExpected1); + + // Check the Backward pass of the layer. + module1.Backward(input1, gy1, g1); + CheckMatrices(g1, gExpected1); + + // Input is a batch of 2 images, each of size (2,2) and having 4 channels. + input2 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 + << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0 + << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr; + gy2 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8 + << 12 << 16 << arma::endr << 17 << 21 << 25 << 29 << 18 << 22 << 26 << 30 + << 19 << 23 << 27 << 31 << 20 << 24 << 28 << 32 << arma::endr; + + // Calculated using torch.nn.PixelShuffle(). + outputExpected2 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0 + << 0 << 0 << 0 << 0 << arma::endr << 5 << 0 << 7 << 0 << 0 << 0 << 0 << 0 + << 6 << 0 << 8 << 0 << 0 << 0 << 0 << 0 << arma::endr; + gExpected2 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12 + << 6 << 14 << 8 << 16 << arma::endr << 17 << 25 << 19 << 27 << 21 << 29 + << 23 << 31 << 18 << 26 << 20 << 28 << 22 << 30 << 24 << 32 << arma::endr; + + input2 = input2.t(); + outputExpected2 = outputExpected2.t(); + gy2 = gy2.t(); + gExpected2 = gExpected2.t(); + + // Check the Forward pass of the layer. + module2.Forward(input2, output2); + CheckMatrices(output2, outputExpected2); + + // Check the Backward pass of the layer. + module2.Backward(input2, gy2, g2); + CheckMatrices(g2, gExpected2); +} + +/** + * Test that the function that can access the parameters of the + * PixelShuffle layer works. + */ +TEST_CASE("PixelShuffleLayerParametersTest", "[ANNLayerTest]") +{ + // Create the layer using the empty constructor. + PixelShuffle<> layer; + + // Set the different input parameters of the layer. + layer.UpscaleFactor() = 2; + layer.InputHeight() = 2; + layer.InputWidth() = 2; + layer.InputChannels() = 4; + + // Make sure we can get the parameters successfully. + REQUIRE(layer.UpscaleFactor() == 2); + REQUIRE(layer.InputHeight() == 2); + REQUIRE(layer.InputWidth() == 2); + REQUIRE(layer.InputChannels() == 4); + + arma::mat input, output; + // Input is a batch of 2 images, each of size (2,2) and having 4 channels. + input << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 + << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0 + << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr; + input = input.t(); + layer.Forward(input, output); + + // Check whether output parameters are returned correctly. + REQUIRE(layer.OutputHeight() == 4); + REQUIRE(layer.OutputWidth() == 4); + REQUIRE(layer.OutputChannels() == 1); +} + +/* * Simple Test for SpatialDropout layer. */ TEST_CASE("SpatialDropoutLayerTest", "[ANNLayerTest]")