Ryan Birmingham
+2
-1
@@ -1,5 +1,7 @@
|
||||
### mlpack ?.?.?
|
||||
###### ????-??-??
|
||||
* Added Pixel Shuffle layer (#2563).
|
||||
|
||||
* Add "check_input_matrices" option to python bindings that checks
|
||||
for NaN and inf values in all the input matrices (#2787).
|
||||
|
||||
@@ -52,7 +54,6 @@
|
||||
|
||||
### mlpack 3.4.0
|
||||
###### 2020-09-01
|
||||
|
||||
* Issue warnings when metrics produce NaNs in KFoldCV (#2595).
|
||||
|
||||
* Added bindings for _R_ during Google Summer of Code (#2556).
|
||||
|
||||
@@ -85,6 +85,8 @@ set(SOURCES
|
||||
noisylinear_impl.hpp
|
||||
parametric_relu.hpp
|
||||
parametric_relu_impl.hpp
|
||||
pixel_shuffle.hpp
|
||||
pixel_shuffle_impl.hpp
|
||||
positional_encoding.hpp
|
||||
positional_encoding_impl.hpp
|
||||
recurrent.hpp
|
||||
|
||||
@@ -58,6 +58,7 @@
|
||||
#include "noisylinear.hpp"
|
||||
#include "padding.hpp"
|
||||
#include "parametric_relu.hpp"
|
||||
#include "pixel_shuffle.hpp"
|
||||
#include "positional_encoding.hpp"
|
||||
#include "recurrent_attention.hpp"
|
||||
#include "recurrent.hpp"
|
||||
|
||||
@@ -43,6 +43,7 @@
|
||||
#include <mlpack/methods/ann/layer/adaptive_max_pooling.hpp>
|
||||
#include <mlpack/methods/ann/layer/adaptive_mean_pooling.hpp>
|
||||
#include <mlpack/methods/ann/layer/parametric_relu.hpp>
|
||||
#include <mlpack/methods/ann/layer/pixel_shuffle.hpp>
|
||||
#include <mlpack/methods/ann/layer/positional_encoding.hpp>
|
||||
#include <mlpack/methods/ann/layer/reinforce_normal.hpp>
|
||||
#include <mlpack/methods/ann/layer/reparametrization.hpp>
|
||||
@@ -222,6 +223,7 @@ class AdaptiveMeanPooling;
|
||||
using MoreTypes = boost::variant<
|
||||
Linear3D<arma::mat, arma::mat, NoRegularizer>*,
|
||||
LpPooling<arma::mat, arma::mat>*,
|
||||
PixelShuffle<arma::mat, arma::mat>*,
|
||||
Glimpse<arma::mat, arma::mat>*,
|
||||
Highway<arma::mat, arma::mat>*,
|
||||
MultiheadAttention<arma::mat, arma::mat, NoRegularizer>*,
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
/**
|
||||
* @file methods/ann/layer/pixel_shuffle.hpp
|
||||
* @author Anjishnu Mukherjee
|
||||
* @author Abhinav Anand
|
||||
*
|
||||
* Definition of the PixelShuffle class.
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_HPP
|
||||
#define MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_HPP
|
||||
|
||||
#include <mlpack/prereqs.hpp>
|
||||
|
||||
namespace mlpack {
|
||||
namespace ann /** Artificial Neural Network. */ {
|
||||
|
||||
/**
|
||||
* Implementation of the PixelShuffle layer.
|
||||
*
|
||||
* For more information, refer to the following paper,
|
||||
*
|
||||
* @code
|
||||
* @article{Shi16,
|
||||
* author = {Wenzhe Shi, Jose Caballero,Ferenc Huszár, Johannes Totz,
|
||||
* Andrew P. Aitken, Rob Bishop, Daniel Rueckert, Zehan Wang},
|
||||
* title = {Real-Time Single Image and Video Super-Resolution Using an
|
||||
* Efficient Sub-Pixel Convolutional Neural Network},
|
||||
* journal = {CoRR},
|
||||
* volume = {abs/1609.05158},
|
||||
* year = {2016},
|
||||
* url = {https://arxiv.org/abs/1609.05158},
|
||||
* eprint = {1609.05158},
|
||||
* }
|
||||
* @endcode
|
||||
*
|
||||
* @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
|
||||
* arma::sp_mat or arma::cube).
|
||||
* @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
|
||||
* arma::sp_mat or arma::cube).
|
||||
*/
|
||||
template <
|
||||
typename InputDataType = arma::mat,
|
||||
typename OutputDataType = arma::mat
|
||||
>
|
||||
class PixelShuffle
|
||||
{
|
||||
public:
|
||||
//! Create the PixelShuffle object.
|
||||
PixelShuffle();
|
||||
|
||||
/**
|
||||
* Create the PixelShuffle object using the specified parameters.
|
||||
* The number of input channels should be an integral multiple of the square
|
||||
* of the upscale factor.
|
||||
*
|
||||
* @param upscaleFactor The scaling factor for Pixel Shuffle.
|
||||
* @param height The height of each input image.
|
||||
* @param width The width of each input image.
|
||||
* @param size The number of channels of each input image.
|
||||
*/
|
||||
PixelShuffle(const size_t upscaleFactor,
|
||||
const size_t height,
|
||||
const size_t width,
|
||||
const size_t size);
|
||||
|
||||
/**
|
||||
* Ordinary feed forward pass of the PixelShuffle layer.
|
||||
*
|
||||
* @param input Input data used for evaluating the specified function.
|
||||
* @param output Resulting output activation.
|
||||
*/
|
||||
template<typename eT>
|
||||
void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);
|
||||
|
||||
/**
|
||||
* Ordinary feed backward pass of the PixelShuffle layer.
|
||||
*
|
||||
* @param input The propagated input activation.
|
||||
* @param gy The backpropagated error.
|
||||
* @param g The calculated gradient.
|
||||
*/
|
||||
template<typename eT>
|
||||
void Backward(const arma::Mat<eT>& input,
|
||||
const arma::Mat<eT>& gy,
|
||||
arma::Mat<eT>& g);
|
||||
|
||||
//! Get the output parameter.
|
||||
OutputDataType const& OutputParameter() const { return outputParameter; }
|
||||
//! Modify the output parameter.
|
||||
OutputDataType& OutputParameter() { return outputParameter; }
|
||||
|
||||
//! Get the delta.
|
||||
OutputDataType const& Delta() const { return delta; }
|
||||
//! Modify the delta.
|
||||
OutputDataType& Delta() { return delta; }
|
||||
|
||||
//! Get the upscale factor.
|
||||
size_t UpscaleFactor() const { return upscaleFactor; }
|
||||
|
||||
//! Modify the upscale factor.
|
||||
size_t& UpscaleFactor() { return upscaleFactor; }
|
||||
|
||||
//! Get the input image height.
|
||||
size_t InputHeight() const { return height; }
|
||||
|
||||
//! Modify the input image height.
|
||||
size_t& InputHeight() { return height; }
|
||||
|
||||
//! Get the input image width.
|
||||
size_t InputWidth() const { return width; }
|
||||
|
||||
//! Modify the input image width.
|
||||
size_t& InputWidth() { return width; }
|
||||
|
||||
//! Get the number of input channels.
|
||||
size_t InputChannels() const { return size; }
|
||||
|
||||
//! Modify the number of input channels.
|
||||
size_t& InputChannels() { return size; }
|
||||
|
||||
//! Get the output image height.
|
||||
size_t OutputHeight() const { return outputHeight; }
|
||||
|
||||
//! Get the output image width.
|
||||
size_t OutputWidth() const { return outputWidth; }
|
||||
|
||||
//! Get the number of output channels.
|
||||
size_t OutputChannels() const { return sizeOut; }
|
||||
|
||||
/**
|
||||
* Serialize the layer.
|
||||
*/
|
||||
template<typename Archive>
|
||||
void serialize(Archive& ar, const unsigned int /* version */);
|
||||
|
||||
private:
|
||||
//! Locally-stored delta object.
|
||||
OutputDataType delta;
|
||||
|
||||
//! Locally-stored output parameter object.
|
||||
OutputDataType outputParameter;
|
||||
|
||||
//! The scaling factor for Pixel Shuffle.
|
||||
size_t upscaleFactor;
|
||||
|
||||
//! The height of each input image.
|
||||
size_t height;
|
||||
|
||||
//! The width of each input image.
|
||||
size_t width;
|
||||
|
||||
//! The number of channels of each input image.
|
||||
size_t size;
|
||||
|
||||
//! The number of images in the batch.
|
||||
size_t batchSize;
|
||||
|
||||
//! The height of each output image.
|
||||
size_t outputHeight;
|
||||
|
||||
//! The width of each output image.
|
||||
size_t outputWidth;
|
||||
|
||||
//! The number of channels of each output image.
|
||||
size_t sizeOut;
|
||||
|
||||
//! A boolean used to do some internal calculations once initially.
|
||||
bool reset;
|
||||
}; // class PixelShuffle
|
||||
|
||||
} // namespace ann
|
||||
} // namespace mlpack
|
||||
|
||||
// Include implementation.
|
||||
#include "pixel_shuffle_impl.hpp"
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,143 @@
|
||||
/**
|
||||
* @file methods/ann/layer/pixel_shuffle_impl.hpp
|
||||
* @author Anjishnu Mukherjee
|
||||
* @author Abhinav Anand
|
||||
*
|
||||
* Implementation of the PixelShuffle class.
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_IMPL_HPP
|
||||
#define MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_IMPL_HPP
|
||||
|
||||
// In case it hasn't yet been included.
|
||||
#include "pixel_shuffle.hpp"
|
||||
|
||||
namespace mlpack {
|
||||
namespace ann /** Artificial Neural Network. */ {
|
||||
|
||||
template<typename InputDataType, typename OutputDataType>
|
||||
PixelShuffle<InputDataType, OutputDataType>::PixelShuffle() :
|
||||
PixelShuffle(0, 0, 0, 0)
|
||||
{
|
||||
// Nothing to do here.
|
||||
}
|
||||
|
||||
template<typename InputDataType, typename OutputDataType>
|
||||
PixelShuffle<InputDataType, OutputDataType>::PixelShuffle(
|
||||
const size_t upscaleFactor,
|
||||
const size_t height,
|
||||
const size_t width,
|
||||
const size_t size) :
|
||||
upscaleFactor(upscaleFactor),
|
||||
height(height),
|
||||
width(width),
|
||||
size(size),
|
||||
batchSize(0),
|
||||
outputHeight(0),
|
||||
outputWidth(0),
|
||||
sizeOut(0),
|
||||
reset(false)
|
||||
{
|
||||
// Nothing to do here.
|
||||
}
|
||||
|
||||
template<typename InputDataType, typename OutputDataType>
|
||||
template<typename eT>
|
||||
void PixelShuffle<InputDataType, OutputDataType>::Forward(
|
||||
const arma::Mat<eT>& input, arma::Mat<eT>& output)
|
||||
{
|
||||
if (!reset)
|
||||
{
|
||||
batchSize = input.n_cols;
|
||||
sizeOut = size / std::pow(upscaleFactor, 2);
|
||||
outputHeight = height * upscaleFactor;
|
||||
outputWidth = width * upscaleFactor;
|
||||
reset = true;
|
||||
}
|
||||
|
||||
output.zeros(outputHeight * outputWidth * sizeOut, batchSize);
|
||||
for (size_t n = 0; n < batchSize; n++)
|
||||
{
|
||||
arma::cube inputTemp(const_cast<arma::mat&>(input).memptr(), height,
|
||||
width, size * batchSize, false, false);
|
||||
arma::cube outputTemp(const_cast<arma::mat&>(output).memptr(),
|
||||
outputHeight, outputWidth, sizeOut * batchSize, false, false);
|
||||
|
||||
for (size_t c = 0; c < sizeOut; c++)
|
||||
{
|
||||
for (size_t h = 0; h < outputHeight; h++)
|
||||
{
|
||||
for (size_t w = 0; w < outputWidth; w++)
|
||||
{
|
||||
size_t height_index = h / upscaleFactor;
|
||||
size_t width_index = w / upscaleFactor;
|
||||
size_t channel_index = (upscaleFactor * (h % upscaleFactor)) +
|
||||
(w % upscaleFactor) + (c * std::pow(upscaleFactor, 2));
|
||||
outputTemp(w, h, c + n * sizeOut) = inputTemp(width_index, height_index,
|
||||
channel_index + n * size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
template<typename InputDataType, typename OutputDataType>
|
||||
template<typename eT>
|
||||
void PixelShuffle<InputDataType, OutputDataType>::Backward(
|
||||
const arma::Mat<eT>& input, const arma::Mat<eT>& gy, arma::Mat<eT>& g)
|
||||
{
|
||||
g.zeros(arma::size(input));
|
||||
for (size_t n = 0; n < batchSize; n++)
|
||||
{
|
||||
arma::cube gyTemp(const_cast<arma::mat&>(gy).memptr(), outputHeight,
|
||||
outputWidth, sizeOut * batchSize, false, false);
|
||||
arma::cube gTemp(const_cast<arma::mat&>(g).memptr(), height, width,
|
||||
size * batchSize, false, false);
|
||||
|
||||
for (size_t c = 0; c < sizeOut; c++)
|
||||
{
|
||||
for (size_t h = 0; h < outputHeight; h++)
|
||||
{
|
||||
for (size_t w = 0; w < outputWidth; w++)
|
||||
{
|
||||
size_t height_index = h / upscaleFactor;
|
||||
size_t width_index = w / upscaleFactor;
|
||||
size_t channel_index = (upscaleFactor * (h % upscaleFactor)) +
|
||||
(w % upscaleFactor) + (c * std::pow(upscaleFactor, 2));
|
||||
gTemp(width_index, height_index, channel_index + n * size) = gyTemp(w, h,
|
||||
c + n * sizeOut);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
template<typename InputDataType, typename OutputDataType>
|
||||
template<typename Archive>
|
||||
void PixelShuffle<InputDataType, OutputDataType>::serialize(
|
||||
Archive& ar,
|
||||
const unsigned int /* version */)
|
||||
{
|
||||
ar(CEREAL_NVP(delta));
|
||||
ar(CEREAL_NVP(outputParameter));
|
||||
ar(CEREAL_NVP(upscaleFactor));
|
||||
ar(CEREAL_NVP(height));
|
||||
ar(CEREAL_NVP(width));
|
||||
ar(CEREAL_NVP(size));
|
||||
ar(CEREAL_NVP(batchSize));
|
||||
ar(CEREAL_NVP(outputHeight));
|
||||
ar(CEREAL_NVP(outputHeight));
|
||||
ar(CEREAL_NVP(outputWidth));
|
||||
ar(CEREAL_NVP(sizeOut));
|
||||
}
|
||||
|
||||
} // namespace ann
|
||||
} // namespace mlpack
|
||||
|
||||
#endif
|
||||
@@ -4606,6 +4606,106 @@ TEST_CASE("TransposedConvolutionWeightInitializationTest", "[ANNLayerTest]")
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple Test for PixelShuffle layer.
|
||||
*/
|
||||
TEST_CASE("PixelShuffleLayerTest", "[ANNLayerTest]")
|
||||
{
|
||||
arma::mat input1, output1, gy1, g1, outputExpected1, gExpected1;
|
||||
arma::mat input2, output2, gy2, g2, outputExpected2, gExpected2;
|
||||
PixelShuffle<> module1(2, 2, 2, 4);
|
||||
PixelShuffle<> module2(2, 2, 2, 4);
|
||||
|
||||
// Input is a single image, of size (2,2) and having 4 channels.
|
||||
input1 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0
|
||||
<< 0 << 0 << arma::endr;
|
||||
gy1 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8
|
||||
<< 12 << 16 << arma::endr;
|
||||
|
||||
// Calculated using torch.nn.PixelShuffle().
|
||||
outputExpected1 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0
|
||||
<< 0 << 0 << 0 << 0 << arma::endr;
|
||||
gExpected1 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12
|
||||
<< 6 << 14 << 8 << 16 << arma::endr;
|
||||
|
||||
input1 = input1.t();
|
||||
outputExpected1 = outputExpected1.t();
|
||||
gy1 = gy1.t();
|
||||
gExpected1 = gExpected1.t();
|
||||
|
||||
// Check the Forward pass of the layer.
|
||||
module1.Forward(input1, output1);
|
||||
CheckMatrices(output1, outputExpected1);
|
||||
|
||||
// Check the Backward pass of the layer.
|
||||
module1.Backward(input1, gy1, g1);
|
||||
CheckMatrices(g1, gExpected1);
|
||||
|
||||
// Input is a batch of 2 images, each of size (2,2) and having 4 channels.
|
||||
input2 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0
|
||||
<< 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0
|
||||
<< 0 << 0 << 0 << 0 << 0 << 0 << arma::endr;
|
||||
gy2 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8
|
||||
<< 12 << 16 << arma::endr << 17 << 21 << 25 << 29 << 18 << 22 << 26 << 30
|
||||
<< 19 << 23 << 27 << 31 << 20 << 24 << 28 << 32 << arma::endr;
|
||||
|
||||
// Calculated using torch.nn.PixelShuffle().
|
||||
outputExpected2 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0
|
||||
<< 0 << 0 << 0 << 0 << arma::endr << 5 << 0 << 7 << 0 << 0 << 0 << 0 << 0
|
||||
<< 6 << 0 << 8 << 0 << 0 << 0 << 0 << 0 << arma::endr;
|
||||
gExpected2 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12
|
||||
<< 6 << 14 << 8 << 16 << arma::endr << 17 << 25 << 19 << 27 << 21 << 29
|
||||
<< 23 << 31 << 18 << 26 << 20 << 28 << 22 << 30 << 24 << 32 << arma::endr;
|
||||
|
||||
input2 = input2.t();
|
||||
outputExpected2 = outputExpected2.t();
|
||||
gy2 = gy2.t();
|
||||
gExpected2 = gExpected2.t();
|
||||
|
||||
// Check the Forward pass of the layer.
|
||||
module2.Forward(input2, output2);
|
||||
CheckMatrices(output2, outputExpected2);
|
||||
|
||||
// Check the Backward pass of the layer.
|
||||
module2.Backward(input2, gy2, g2);
|
||||
CheckMatrices(g2, gExpected2);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that the function that can access the parameters of the
|
||||
* PixelShuffle layer works.
|
||||
*/
|
||||
TEST_CASE("PixelShuffleLayerParametersTest", "[ANNLayerTest]")
|
||||
{
|
||||
// Create the layer using the empty constructor.
|
||||
PixelShuffle<> layer;
|
||||
|
||||
// Set the different input parameters of the layer.
|
||||
layer.UpscaleFactor() = 2;
|
||||
layer.InputHeight() = 2;
|
||||
layer.InputWidth() = 2;
|
||||
layer.InputChannels() = 4;
|
||||
|
||||
// Make sure we can get the parameters successfully.
|
||||
REQUIRE(layer.UpscaleFactor() == 2);
|
||||
REQUIRE(layer.InputHeight() == 2);
|
||||
REQUIRE(layer.InputWidth() == 2);
|
||||
REQUIRE(layer.InputChannels() == 4);
|
||||
|
||||
arma::mat input, output;
|
||||
// Input is a batch of 2 images, each of size (2,2) and having 4 channels.
|
||||
input << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0
|
||||
<< 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0
|
||||
<< 0 << 0 << 0 << 0 << 0 << 0 << arma::endr;
|
||||
input = input.t();
|
||||
layer.Forward(input, output);
|
||||
|
||||
// Check whether output parameters are returned correctly.
|
||||
REQUIRE(layer.OutputHeight() == 4);
|
||||
REQUIRE(layer.OutputWidth() == 4);
|
||||
REQUIRE(layer.OutputChannels() == 1);
|
||||
}
|
||||
|
||||
/*
|
||||
* Simple Test for SpatialDropout layer.
|
||||
*/
|
||||
TEST_CASE("SpatialDropoutLayerTest", "[ANNLayerTest]")
|
||||
|
||||
Reference in New Issue
Block a user