Merge branch 'master' into multilabel_softmargin_loss
This commit is contained in:
jeffin sam
+2
-1
@@ -2,6 +2,8 @@
|
||||
###### ????-??-??
|
||||
* Added `Multi Label Soft Margin Loss` loss function for neural networks
|
||||
(#2345).
|
||||
|
||||
* Added Pixel Shuffle layer (#2563).
|
||||
|
||||
* Add "check_input_matrices" option to python bindings that checks
|
||||
for NaN and inf values in all the input matrices (#2787).
|
||||
@@ -55,7 +57,6 @@
|
||||
|
||||
### mlpack 3.4.0
|
||||
###### 2020-09-01
|
||||
|
||||
* Issue warnings when metrics produce NaNs in KFoldCV (#2595).
|
||||
|
||||
* Added bindings for _R_ during Google Summer of Code (#2556).
|
||||
|
||||
@@ -19,6 +19,7 @@ set(SOURCES
|
||||
multi_quadratic_function.hpp
|
||||
poisson1_function.hpp
|
||||
gaussian_function.hpp
|
||||
hard_swish_function.hpp
|
||||
)
|
||||
|
||||
# Add directory name to sources.
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
/**
|
||||
* @file methods/ann/activation_functions/hard_swish_function.hpp
|
||||
* @author Anush Kini
|
||||
*
|
||||
* Definition and implementation of the Hard Swish function as described by
|
||||
* Howard A, Sandler M, Chu G, Chen LC, Chen B, Tan M, Wang W, Zhu Y, Pang R,
|
||||
* Vasudevan V and Le QV.
|
||||
* For more information, see the following paper.
|
||||
*
|
||||
* @code
|
||||
* @misc{
|
||||
* author = {Howard A, Sandler M, Chu G, Chen LC, Chen B, Tan M, Wang W,
|
||||
* Zhu Y, Pang R, Vasudevan V and Le QV},
|
||||
* title = {Searching for MobileNetV3},
|
||||
* year = {2019}
|
||||
* }
|
||||
* @endcode
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_HARD_SWISH_FUNCTION_HPP
|
||||
#define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_HARD_SWISH_FUNCTION_HPP
|
||||
|
||||
#include <mlpack/prereqs.hpp>
|
||||
|
||||
namespace mlpack {
|
||||
namespace ann /** Artificial Neural Network. */ {
|
||||
/**
|
||||
* The Hard Swish function, defined by
|
||||
*
|
||||
* @f{eqnarray*}{
|
||||
* f(x) &=& \begin{cases}
|
||||
* 0 & x \leq -3\\
|
||||
* x & x \geq +3\\
|
||||
* \frac{x * (x + 3)}{6} & otherwise\\
|
||||
* \end{cases} \\
|
||||
* f'(x) &=& \begin{cases}
|
||||
* 0 & x \leq -3\\
|
||||
* 1 & x \geq +3\\
|
||||
* \frac{2x + 3}{6} & otherwise\\
|
||||
* \end{cases}
|
||||
* @f}
|
||||
*/
|
||||
class HardSwishFunction
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* Computes the Hard Swish function.
|
||||
*
|
||||
* @param x Input data.
|
||||
* @return f(x).
|
||||
*/
|
||||
static double Fn(const double x)
|
||||
{
|
||||
if (x <= -3)
|
||||
return 0;
|
||||
else if (x >= 3)
|
||||
return x;
|
||||
|
||||
return x * (x + 3) / 6;
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the Hard Swish function.
|
||||
*
|
||||
* @param x Input data.
|
||||
* @param y The resulting output activation.
|
||||
*/
|
||||
template <typename InputVecType, typename OutputVecType>
|
||||
static void Fn(const InputVecType &x, OutputVecType &y)
|
||||
{
|
||||
y.set_size(size(x));
|
||||
|
||||
for (size_t i = 0; i < x.n_elem; i++)
|
||||
y(i) = Fn(x(i));
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the first derivative of the Hard Swish function.
|
||||
*
|
||||
* @param y Input data.
|
||||
* @return f'(x).
|
||||
*/
|
||||
static double Deriv(const double y)
|
||||
{
|
||||
if (y <= -3)
|
||||
return 0;
|
||||
else if (y >= 3)
|
||||
return 1;
|
||||
|
||||
return (2 * y + 3.0) / 6.0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the first derivatives of the Hard Swish function.
|
||||
*
|
||||
* @param y Input data.
|
||||
* @param x The resulting derivatives.
|
||||
*/
|
||||
template <typename InputVecType, typename OutputVecType>
|
||||
static void Deriv(const InputVecType &y, OutputVecType &x)
|
||||
{
|
||||
x.set_size(size(y));
|
||||
|
||||
for (size_t i = 0; i < y.n_elem; i++)
|
||||
x(i) = Deriv(y(i));
|
||||
}
|
||||
}; // class HardSwishFunction
|
||||
|
||||
} // namespace ann
|
||||
} // namespace mlpack
|
||||
|
||||
#endif
|
||||
@@ -46,6 +46,8 @@ set(SOURCES
|
||||
hard_tanh_impl.hpp
|
||||
highway.hpp
|
||||
highway_impl.hpp
|
||||
isrlu.hpp
|
||||
isrlu_impl.hpp
|
||||
join.hpp
|
||||
join_impl.hpp
|
||||
layer.hpp
|
||||
@@ -83,6 +85,8 @@ set(SOURCES
|
||||
noisylinear_impl.hpp
|
||||
parametric_relu.hpp
|
||||
parametric_relu_impl.hpp
|
||||
pixel_shuffle.hpp
|
||||
pixel_shuffle_impl.hpp
|
||||
positional_encoding.hpp
|
||||
positional_encoding_impl.hpp
|
||||
recurrent.hpp
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
#include <mlpack/methods/ann/activation_functions/elliot_function.hpp>
|
||||
#include <mlpack/methods/ann/activation_functions/elish_function.hpp>
|
||||
#include <mlpack/methods/ann/activation_functions/gaussian_function.hpp>
|
||||
#include <mlpack/methods/ann/activation_functions/hard_swish_function.hpp>
|
||||
|
||||
namespace mlpack {
|
||||
namespace ann /** Artificial Neural Network. */ {
|
||||
@@ -50,6 +51,7 @@ namespace ann /** Artificial Neural Network. */ {
|
||||
* - ELiSHLayer
|
||||
* - ElliotLayer
|
||||
* - GaussianLayer
|
||||
* - HardSwishLayer
|
||||
*
|
||||
* @tparam ActivationFunction Activation function used for the embedding layer.
|
||||
* @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
|
||||
@@ -277,6 +279,17 @@ template <
|
||||
using GaussianFunctionLayer = BaseLayer<
|
||||
ActivationFunction, InputDataType, OutputDataType>;
|
||||
|
||||
/**
|
||||
* Standard HardSwish-Layer using the HardSwish activation function.
|
||||
*/
|
||||
template <
|
||||
class ActivationFunction = HardSwishFunction,
|
||||
typename InputDataType = arma::mat,
|
||||
typename OutputDataType = arma::mat
|
||||
>
|
||||
using HardSwishFunctionLayer = BaseLayer<
|
||||
ActivationFunction, InputDataType, OutputDataType>;
|
||||
|
||||
} // namespace ann
|
||||
} // namespace mlpack
|
||||
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
/**
|
||||
* @file methods/ann/layer/isrlu.hpp
|
||||
* @author Abhinav Anand
|
||||
*
|
||||
* Definition of the ISRLU activation function as described by Jonathan T. Barron.
|
||||
*
|
||||
* For more information, read the following paper.
|
||||
*
|
||||
* @code
|
||||
* @article{
|
||||
* author = {Carlile, Brad and Delamarter, Guy and Kinney, Paul and Marti,
|
||||
* Akiko and Whitney, Brian},
|
||||
* title = {Improving deep learning by inverse square root linear units (ISRLUs)},
|
||||
* year = {2017},
|
||||
* url = {https://arxiv.org/pdf/1710.09967.pdf}
|
||||
* }
|
||||
* @endcode
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef MLPACK_METHODS_ANN_LAYER_ISRLU_HPP
|
||||
#define MLPACK_METHODS_ANN_LAYER_ISRLU_HPP
|
||||
|
||||
#include <mlpack/prereqs.hpp>
|
||||
|
||||
namespace mlpack {
|
||||
namespace ann /** Artificial Neural Network. */ {
|
||||
|
||||
/**
|
||||
* The ISRLU activation function, defined by
|
||||
*
|
||||
* @f{eqnarray*}{
|
||||
* f(x) &=& \left\{
|
||||
* \begin{array}{lr}
|
||||
* x & : x \ge 0 \\
|
||||
* x(\frac{1}{1 + \alpha x^2}) & : x < 0
|
||||
* \end{array}
|
||||
* \right. \\
|
||||
* f'(x) &=& \left\{
|
||||
* \begin{array}{lr}
|
||||
* x & : 1 \ge 0 \\
|
||||
* (\frac{1}{1 + \alpha x^2})^3 & : x < 0
|
||||
* \end{array}
|
||||
* \right.
|
||||
* @f}
|
||||
*
|
||||
*
|
||||
* @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
|
||||
* arma::sp_mat or arma::cube).
|
||||
* @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
|
||||
* arma::sp_mat or arma::cube).
|
||||
*/
|
||||
template <
|
||||
typename InputDataType = arma::mat,
|
||||
typename OutputDataType = arma::mat
|
||||
>
|
||||
class ISRLU
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* Create the ISRLU object using the specified parameter.
|
||||
*
|
||||
* @param alpha Scale parameter controls the value to which an ISRLU
|
||||
* saturates for negative inputs.
|
||||
*/
|
||||
ISRLU(const double alpha = 1.0);
|
||||
|
||||
/**
|
||||
* Ordinary feed forward pass of a neural network, evaluating the function
|
||||
* f(x) by propagating the activity forward through f.
|
||||
*
|
||||
* @param input Input data used for evaluating the specified function.
|
||||
* @param output Resulting output activation.
|
||||
*/
|
||||
template<typename InputType, typename OutputType>
|
||||
void Forward(const InputType& input, OutputType& output);
|
||||
|
||||
/**
|
||||
* Ordinary feed backward pass of a neural network, calculating the function
|
||||
* f(x) by propagating x backwards through f. Using the results from the feed
|
||||
* forward pass.
|
||||
*
|
||||
* @param input The propagated input activation f(x).
|
||||
* @param gy The backpropagated error.
|
||||
* @param g The calculated gradient.
|
||||
*/
|
||||
template<typename DataType>
|
||||
void Backward(const DataType& input, const DataType& gy, DataType& g);
|
||||
|
||||
//! Get the output parameter.
|
||||
OutputDataType const& OutputParameter() const { return outputParameter; }
|
||||
//! Modify the output parameter.
|
||||
OutputDataType& OutputParameter() { return outputParameter; }
|
||||
|
||||
//! Get the delta.
|
||||
OutputDataType const& Delta() const { return delta; }
|
||||
//! Modify the delta.
|
||||
OutputDataType& Delta() { return delta; }
|
||||
|
||||
//! Get the non zero gradient.
|
||||
double const& Alpha() const { return alpha; }
|
||||
//! Modify the non zero gradient.
|
||||
double& Alpha() { return alpha; }
|
||||
|
||||
//! Get size of weights.
|
||||
size_t WeightSize() { return 0; }
|
||||
|
||||
/**
|
||||
* Serialize the layer.
|
||||
*/
|
||||
template<typename Archive>
|
||||
void serialize(Archive& ar, const uint32_t /* version */);
|
||||
|
||||
private:
|
||||
//! Locally-stored delta object.
|
||||
OutputDataType delta;
|
||||
|
||||
//! Locally-stored output parameter object.
|
||||
OutputDataType outputParameter;
|
||||
|
||||
//! Locally stored first derivative of the activation function.
|
||||
arma::mat derivative;
|
||||
|
||||
//! ISRLU Hyperparameter (alpha > 0).
|
||||
double alpha;
|
||||
|
||||
}; // class ISRLU
|
||||
|
||||
} // namespace ann
|
||||
} // namespace mlpack
|
||||
|
||||
// Include implementation.
|
||||
#include "isrlu_impl.hpp"
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* @file methods/ann/layer/isrlu_impl.hpp
|
||||
* @author Abhinav Anand
|
||||
*
|
||||
* Implementation of the ISRLU activation function as described by Jonathan T. Barron.
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef MLPACK_METHODS_ANN_LAYER_ISRLU_IMPL_HPP
|
||||
#define MLPACK_METHODS_ANN_LAYER_ISRLU_IMPL_HPP
|
||||
|
||||
// In case it hasn't yet been included.
|
||||
#include "isrlu.hpp"
|
||||
|
||||
namespace mlpack {
|
||||
namespace ann /** Artificial Neural Network. */ {
|
||||
|
||||
template<typename InputDataType, typename OutputDataType>
|
||||
ISRLU<InputDataType, OutputDataType>::ISRLU(const double alpha) :
|
||||
alpha(alpha)
|
||||
{}
|
||||
|
||||
template<typename InputDataType, typename OutputDataType>
|
||||
template<typename InputType, typename OutputType>
|
||||
void ISRLU<InputDataType, OutputDataType>::Forward(
|
||||
const InputType& input, OutputType& output)
|
||||
{
|
||||
output = arma::ones<OutputDataType>(arma::size(input));
|
||||
for (size_t i = 0; i < input.n_elem; ++i)
|
||||
{
|
||||
output(i) = (input(i) >= 0) ? input(i) : input(i) *
|
||||
(1 / std::sqrt(1 + alpha * (input(i) * input(i))));
|
||||
}
|
||||
}
|
||||
|
||||
template<typename InputDataType, typename OutputDataType>
|
||||
template<typename DataType>
|
||||
void ISRLU<InputDataType, OutputDataType>::Backward(
|
||||
const DataType& input, const DataType& gy, DataType& g)
|
||||
{
|
||||
derivative.set_size(arma::size(input));
|
||||
for (size_t i = 0; i < input.n_elem; ++i)
|
||||
{
|
||||
derivative(i) = (input(i) >= 0) ? 1 :
|
||||
std::pow(1 / std::sqrt(1 + alpha * input(i) * input(i)), 3);
|
||||
}
|
||||
g = gy % derivative;
|
||||
}
|
||||
|
||||
template<typename InputDataType, typename OutputDataType>
|
||||
template<typename Archive>
|
||||
void ISRLU<InputDataType, OutputDataType>::serialize(
|
||||
Archive& ar,
|
||||
const uint32_t /* version */)
|
||||
{
|
||||
ar(CEREAL_NVP(alpha));
|
||||
}
|
||||
|
||||
} // namespace ann
|
||||
} // namespace mlpack
|
||||
|
||||
#endif
|
||||
@@ -58,6 +58,7 @@
|
||||
#include "noisylinear.hpp"
|
||||
#include "padding.hpp"
|
||||
#include "parametric_relu.hpp"
|
||||
#include "pixel_shuffle.hpp"
|
||||
#include "positional_encoding.hpp"
|
||||
#include "recurrent_attention.hpp"
|
||||
#include "recurrent.hpp"
|
||||
|
||||
@@ -43,6 +43,7 @@
|
||||
#include <mlpack/methods/ann/layer/adaptive_max_pooling.hpp>
|
||||
#include <mlpack/methods/ann/layer/adaptive_mean_pooling.hpp>
|
||||
#include <mlpack/methods/ann/layer/parametric_relu.hpp>
|
||||
#include <mlpack/methods/ann/layer/pixel_shuffle.hpp>
|
||||
#include <mlpack/methods/ann/layer/positional_encoding.hpp>
|
||||
#include <mlpack/methods/ann/layer/reinforce_normal.hpp>
|
||||
#include <mlpack/methods/ann/layer/reparametrization.hpp>
|
||||
@@ -53,6 +54,7 @@
|
||||
#include <mlpack/methods/ann/layer/virtual_batch_norm.hpp>
|
||||
#include <mlpack/methods/ann/layer/hardshrink.hpp>
|
||||
#include <mlpack/methods/ann/layer/celu.hpp>
|
||||
#include <mlpack/methods/ann/layer/isrlu.hpp>
|
||||
#include <mlpack/methods/ann/layer/softshrink.hpp>
|
||||
#include <mlpack/methods/ann/layer/radial_basis_function.hpp>
|
||||
|
||||
@@ -221,6 +223,7 @@ class AdaptiveMeanPooling;
|
||||
using MoreTypes = boost::variant<
|
||||
Linear3D<arma::mat, arma::mat, NoRegularizer>*,
|
||||
LpPooling<arma::mat, arma::mat>*,
|
||||
PixelShuffle<arma::mat, arma::mat>*,
|
||||
Glimpse<arma::mat, arma::mat>*,
|
||||
Highway<arma::mat, arma::mat>*,
|
||||
MultiheadAttention<arma::mat, arma::mat, NoRegularizer>*,
|
||||
@@ -236,7 +239,8 @@ using MoreTypes = boost::variant<
|
||||
VirtualBatchNorm<arma::mat, arma::mat>*,
|
||||
RBF<arma::mat, arma::mat, GaussianFunction>*,
|
||||
BaseLayer<GaussianFunction, arma::mat, arma::mat>*,
|
||||
PositionalEncoding<arma::mat, arma::mat>*
|
||||
PositionalEncoding<arma::mat, arma::mat>*,
|
||||
ISRLU<arma::mat, arma::mat>*
|
||||
>;
|
||||
|
||||
template <typename... CustomLayers>
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
/**
|
||||
* @file methods/ann/layer/pixel_shuffle.hpp
|
||||
* @author Anjishnu Mukherjee
|
||||
* @author Abhinav Anand
|
||||
*
|
||||
* Definition of the PixelShuffle class.
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_HPP
|
||||
#define MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_HPP
|
||||
|
||||
#include <mlpack/prereqs.hpp>
|
||||
|
||||
namespace mlpack {
|
||||
namespace ann /** Artificial Neural Network. */ {
|
||||
|
||||
/**
|
||||
* Implementation of the PixelShuffle layer.
|
||||
*
|
||||
* For more information, refer to the following paper,
|
||||
*
|
||||
* @code
|
||||
* @article{Shi16,
|
||||
* author = {Wenzhe Shi, Jose Caballero,Ferenc Huszár, Johannes Totz,
|
||||
* Andrew P. Aitken, Rob Bishop, Daniel Rueckert, Zehan Wang},
|
||||
* title = {Real-Time Single Image and Video Super-Resolution Using an
|
||||
* Efficient Sub-Pixel Convolutional Neural Network},
|
||||
* journal = {CoRR},
|
||||
* volume = {abs/1609.05158},
|
||||
* year = {2016},
|
||||
* url = {https://arxiv.org/abs/1609.05158},
|
||||
* eprint = {1609.05158},
|
||||
* }
|
||||
* @endcode
|
||||
*
|
||||
* @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
|
||||
* arma::sp_mat or arma::cube).
|
||||
* @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
|
||||
* arma::sp_mat or arma::cube).
|
||||
*/
|
||||
template <
|
||||
typename InputDataType = arma::mat,
|
||||
typename OutputDataType = arma::mat
|
||||
>
|
||||
class PixelShuffle
|
||||
{
|
||||
public:
|
||||
//! Create the PixelShuffle object.
|
||||
PixelShuffle();
|
||||
|
||||
/**
|
||||
* Create the PixelShuffle object using the specified parameters.
|
||||
* The number of input channels should be an integral multiple of the square
|
||||
* of the upscale factor.
|
||||
*
|
||||
* @param upscaleFactor The scaling factor for Pixel Shuffle.
|
||||
* @param height The height of each input image.
|
||||
* @param width The width of each input image.
|
||||
* @param size The number of channels of each input image.
|
||||
*/
|
||||
PixelShuffle(const size_t upscaleFactor,
|
||||
const size_t height,
|
||||
const size_t width,
|
||||
const size_t size);
|
||||
|
||||
/**
|
||||
* Ordinary feed forward pass of the PixelShuffle layer.
|
||||
*
|
||||
* @param input Input data used for evaluating the specified function.
|
||||
* @param output Resulting output activation.
|
||||
*/
|
||||
template<typename eT>
|
||||
void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);
|
||||
|
||||
/**
|
||||
* Ordinary feed backward pass of the PixelShuffle layer.
|
||||
*
|
||||
* @param input The propagated input activation.
|
||||
* @param gy The backpropagated error.
|
||||
* @param g The calculated gradient.
|
||||
*/
|
||||
template<typename eT>
|
||||
void Backward(const arma::Mat<eT>& input,
|
||||
const arma::Mat<eT>& gy,
|
||||
arma::Mat<eT>& g);
|
||||
|
||||
//! Get the output parameter.
|
||||
OutputDataType const& OutputParameter() const { return outputParameter; }
|
||||
//! Modify the output parameter.
|
||||
OutputDataType& OutputParameter() { return outputParameter; }
|
||||
|
||||
//! Get the delta.
|
||||
OutputDataType const& Delta() const { return delta; }
|
||||
//! Modify the delta.
|
||||
OutputDataType& Delta() { return delta; }
|
||||
|
||||
//! Get the upscale factor.
|
||||
size_t UpscaleFactor() const { return upscaleFactor; }
|
||||
|
||||
//! Modify the upscale factor.
|
||||
size_t& UpscaleFactor() { return upscaleFactor; }
|
||||
|
||||
//! Get the input image height.
|
||||
size_t InputHeight() const { return height; }
|
||||
|
||||
//! Modify the input image height.
|
||||
size_t& InputHeight() { return height; }
|
||||
|
||||
//! Get the input image width.
|
||||
size_t InputWidth() const { return width; }
|
||||
|
||||
//! Modify the input image width.
|
||||
size_t& InputWidth() { return width; }
|
||||
|
||||
//! Get the number of input channels.
|
||||
size_t InputChannels() const { return size; }
|
||||
|
||||
//! Modify the number of input channels.
|
||||
size_t& InputChannels() { return size; }
|
||||
|
||||
//! Get the output image height.
|
||||
size_t OutputHeight() const { return outputHeight; }
|
||||
|
||||
//! Get the output image width.
|
||||
size_t OutputWidth() const { return outputWidth; }
|
||||
|
||||
//! Get the number of output channels.
|
||||
size_t OutputChannels() const { return sizeOut; }
|
||||
|
||||
/**
|
||||
* Serialize the layer.
|
||||
*/
|
||||
template<typename Archive>
|
||||
void serialize(Archive& ar, const unsigned int /* version */);
|
||||
|
||||
private:
|
||||
//! Locally-stored delta object.
|
||||
OutputDataType delta;
|
||||
|
||||
//! Locally-stored output parameter object.
|
||||
OutputDataType outputParameter;
|
||||
|
||||
//! The scaling factor for Pixel Shuffle.
|
||||
size_t upscaleFactor;
|
||||
|
||||
//! The height of each input image.
|
||||
size_t height;
|
||||
|
||||
//! The width of each input image.
|
||||
size_t width;
|
||||
|
||||
//! The number of channels of each input image.
|
||||
size_t size;
|
||||
|
||||
//! The number of images in the batch.
|
||||
size_t batchSize;
|
||||
|
||||
//! The height of each output image.
|
||||
size_t outputHeight;
|
||||
|
||||
//! The width of each output image.
|
||||
size_t outputWidth;
|
||||
|
||||
//! The number of channels of each output image.
|
||||
size_t sizeOut;
|
||||
|
||||
//! A boolean used to do some internal calculations once initially.
|
||||
bool reset;
|
||||
}; // class PixelShuffle
|
||||
|
||||
} // namespace ann
|
||||
} // namespace mlpack
|
||||
|
||||
// Include implementation.
|
||||
#include "pixel_shuffle_impl.hpp"
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,143 @@
|
||||
/**
|
||||
* @file methods/ann/layer/pixel_shuffle_impl.hpp
|
||||
* @author Anjishnu Mukherjee
|
||||
* @author Abhinav Anand
|
||||
*
|
||||
* Implementation of the PixelShuffle class.
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_IMPL_HPP
|
||||
#define MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_IMPL_HPP
|
||||
|
||||
// In case it hasn't yet been included.
|
||||
#include "pixel_shuffle.hpp"
|
||||
|
||||
namespace mlpack {
|
||||
namespace ann /** Artificial Neural Network. */ {
|
||||
|
||||
template<typename InputDataType, typename OutputDataType>
|
||||
PixelShuffle<InputDataType, OutputDataType>::PixelShuffle() :
|
||||
PixelShuffle(0, 0, 0, 0)
|
||||
{
|
||||
// Nothing to do here.
|
||||
}
|
||||
|
||||
template<typename InputDataType, typename OutputDataType>
|
||||
PixelShuffle<InputDataType, OutputDataType>::PixelShuffle(
|
||||
const size_t upscaleFactor,
|
||||
const size_t height,
|
||||
const size_t width,
|
||||
const size_t size) :
|
||||
upscaleFactor(upscaleFactor),
|
||||
height(height),
|
||||
width(width),
|
||||
size(size),
|
||||
batchSize(0),
|
||||
outputHeight(0),
|
||||
outputWidth(0),
|
||||
sizeOut(0),
|
||||
reset(false)
|
||||
{
|
||||
// Nothing to do here.
|
||||
}
|
||||
|
||||
template<typename InputDataType, typename OutputDataType>
|
||||
template<typename eT>
|
||||
void PixelShuffle<InputDataType, OutputDataType>::Forward(
|
||||
const arma::Mat<eT>& input, arma::Mat<eT>& output)
|
||||
{
|
||||
if (!reset)
|
||||
{
|
||||
batchSize = input.n_cols;
|
||||
sizeOut = size / std::pow(upscaleFactor, 2);
|
||||
outputHeight = height * upscaleFactor;
|
||||
outputWidth = width * upscaleFactor;
|
||||
reset = true;
|
||||
}
|
||||
|
||||
output.zeros(outputHeight * outputWidth * sizeOut, batchSize);
|
||||
for (size_t n = 0; n < batchSize; n++)
|
||||
{
|
||||
arma::cube inputTemp(const_cast<arma::mat&>(input).memptr(), height,
|
||||
width, size * batchSize, false, false);
|
||||
arma::cube outputTemp(const_cast<arma::mat&>(output).memptr(),
|
||||
outputHeight, outputWidth, sizeOut * batchSize, false, false);
|
||||
|
||||
for (size_t c = 0; c < sizeOut; c++)
|
||||
{
|
||||
for (size_t h = 0; h < outputHeight; h++)
|
||||
{
|
||||
for (size_t w = 0; w < outputWidth; w++)
|
||||
{
|
||||
size_t height_index = h / upscaleFactor;
|
||||
size_t width_index = w / upscaleFactor;
|
||||
size_t channel_index = (upscaleFactor * (h % upscaleFactor)) +
|
||||
(w % upscaleFactor) + (c * std::pow(upscaleFactor, 2));
|
||||
outputTemp(w, h, c + n * sizeOut) = inputTemp(width_index, height_index,
|
||||
channel_index + n * size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
template<typename InputDataType, typename OutputDataType>
|
||||
template<typename eT>
|
||||
void PixelShuffle<InputDataType, OutputDataType>::Backward(
|
||||
const arma::Mat<eT>& input, const arma::Mat<eT>& gy, arma::Mat<eT>& g)
|
||||
{
|
||||
g.zeros(arma::size(input));
|
||||
for (size_t n = 0; n < batchSize; n++)
|
||||
{
|
||||
arma::cube gyTemp(const_cast<arma::mat&>(gy).memptr(), outputHeight,
|
||||
outputWidth, sizeOut * batchSize, false, false);
|
||||
arma::cube gTemp(const_cast<arma::mat&>(g).memptr(), height, width,
|
||||
size * batchSize, false, false);
|
||||
|
||||
for (size_t c = 0; c < sizeOut; c++)
|
||||
{
|
||||
for (size_t h = 0; h < outputHeight; h++)
|
||||
{
|
||||
for (size_t w = 0; w < outputWidth; w++)
|
||||
{
|
||||
size_t height_index = h / upscaleFactor;
|
||||
size_t width_index = w / upscaleFactor;
|
||||
size_t channel_index = (upscaleFactor * (h % upscaleFactor)) +
|
||||
(w % upscaleFactor) + (c * std::pow(upscaleFactor, 2));
|
||||
gTemp(width_index, height_index, channel_index + n * size) = gyTemp(w, h,
|
||||
c + n * sizeOut);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
template<typename InputDataType, typename OutputDataType>
|
||||
template<typename Archive>
|
||||
void PixelShuffle<InputDataType, OutputDataType>::serialize(
|
||||
Archive& ar,
|
||||
const unsigned int /* version */)
|
||||
{
|
||||
ar(CEREAL_NVP(delta));
|
||||
ar(CEREAL_NVP(outputParameter));
|
||||
ar(CEREAL_NVP(upscaleFactor));
|
||||
ar(CEREAL_NVP(height));
|
||||
ar(CEREAL_NVP(width));
|
||||
ar(CEREAL_NVP(size));
|
||||
ar(CEREAL_NVP(batchSize));
|
||||
ar(CEREAL_NVP(outputHeight));
|
||||
ar(CEREAL_NVP(outputHeight));
|
||||
ar(CEREAL_NVP(outputWidth));
|
||||
ar(CEREAL_NVP(sizeOut));
|
||||
}
|
||||
|
||||
} // namespace ann
|
||||
} // namespace mlpack
|
||||
|
||||
#endif
|
||||
@@ -34,6 +34,7 @@ add_executable(mlpack_test
|
||||
facilities_test.cpp
|
||||
fastmks_test.cpp
|
||||
feedforward_network_test.cpp
|
||||
feedforward_network_2_test.cpp
|
||||
gan_test.cpp
|
||||
gmm_test.cpp
|
||||
hmm_test.cpp
|
||||
|
||||
@@ -32,6 +32,7 @@
|
||||
#include <mlpack/methods/ann/activation_functions/spline_function.hpp>
|
||||
#include <mlpack/methods/ann/activation_functions/poisson1_function.hpp>
|
||||
#include <mlpack/methods/ann/activation_functions/gaussian_function.hpp>
|
||||
#include <mlpack/methods/ann/activation_functions/hard_swish_function.hpp>
|
||||
|
||||
#include "catch.hpp"
|
||||
|
||||
@@ -558,6 +559,54 @@ void CheckCELUDerivativeCorrect(const arma::colvec input,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Implementation of the ISRLU activation function test. The function is
|
||||
* implemented as ISRLU layer in the file isrlu.hpp.
|
||||
*
|
||||
* @param input Input data used for evaluating the ISRLU activation function.
|
||||
* @param target Target data used to evaluate the ISRLU activation.
|
||||
*/
|
||||
void CheckISRLUActivationCorrect(const arma::colvec input,
|
||||
const arma::colvec target)
|
||||
{
|
||||
// Initialize ISRLU object with alpha = 1.0.
|
||||
ISRLU<> lrf(1.0);
|
||||
|
||||
// Test the activation function using the entire vector as input.
|
||||
arma::colvec activations;
|
||||
lrf.Forward(input, activations);
|
||||
for (size_t i = 0; i < activations.n_elem; ++i)
|
||||
{
|
||||
REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Implementation of the ISRLU activation function derivative test. The function
|
||||
* is implemented as ISRLU layer in the file isrlu.hpp.
|
||||
*
|
||||
* @param input Input data used for evaluating the ISRLU activation function.
|
||||
* @param target Target data used to evaluate the ISRLU activation.
|
||||
*/
|
||||
void CheckISRLUDerivativeCorrect(const arma::colvec input,
|
||||
const arma::colvec target)
|
||||
{
|
||||
// Initialize ISRLU object with alpha = 1.0.
|
||||
ISRLU<> lrf(1.0);
|
||||
|
||||
// Test the calculation of the derivatives using the entire vector as input.
|
||||
arma::colvec derivatives, activations;
|
||||
|
||||
// This error vector will be set to 1 to get the derivatives.
|
||||
arma::colvec error = arma::ones<arma::colvec>(input.n_elem);
|
||||
lrf.Forward(input, activations);
|
||||
lrf.Backward(activations, error, derivatives);
|
||||
for (size_t i = 0; i < derivatives.n_elem; ++i)
|
||||
{
|
||||
REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Implementation of the Softmin activation function test. The function is
|
||||
* implemented as Softmin layer in the file softmin.hpp.
|
||||
@@ -991,6 +1040,22 @@ TEST_CASE("CELUFunctionTest", "[ActivationFunctionsTest]")
|
||||
CheckCELUDerivativeCorrect(desiredActivations, desiredDerivatives);
|
||||
}
|
||||
|
||||
/**
|
||||
* Basic test of the ISRLU activation function.
|
||||
*/
|
||||
TEST_CASE("ISRLUFunctionTest", "[ActivationFunctionsTest]")
|
||||
{
|
||||
const arma::colvec desiredActivations("-0.89442719 3.2 4.5 \
|
||||
-0.99995020 1 -0.70710678 2 0");
|
||||
|
||||
const arma::colvec desiredDerivatives("0.41408666 1 1 \
|
||||
0.35357980 1 \
|
||||
0.54433105 1 1");
|
||||
|
||||
CheckISRLUActivationCorrect(activationData, desiredActivations);
|
||||
CheckISRLUDerivativeCorrect(activationData, desiredDerivatives);
|
||||
}
|
||||
|
||||
/**
|
||||
* Basic test of the inverse quadratic function.
|
||||
*/
|
||||
@@ -1134,3 +1199,24 @@ TEST_CASE("SoftminFunctionTest", "[ActivationFunctionsTest]")
|
||||
CheckSoftminDerivativeCorrect(activationData,
|
||||
desiredDerivatives);
|
||||
}
|
||||
|
||||
/**
|
||||
* Basic test of the Hard Swish function.
|
||||
*/
|
||||
TEST_CASE("HardSwishFunctionTest", "[ActivationFunctionsTest]")
|
||||
{
|
||||
// Randomly generated data.
|
||||
const arma::colvec activationData("3.6544 -1.9714 -5.2277 1.5448 2.1164");
|
||||
|
||||
// Hand-calculated values.
|
||||
const arma::colvec desiredActivations("3.6544 -0.3379636 0.0 \
|
||||
1.1701345 1.8047248");
|
||||
|
||||
// Hand-calculated values.
|
||||
const arma::colvec desiredDerivatives("1.0 0.38734546 0.5 \
|
||||
0.89004483 1.1015749");
|
||||
|
||||
CheckActivationCorrect<HardSwishFunction>(activationData, desiredActivations);
|
||||
CheckDerivativeCorrect<HardSwishFunction>
|
||||
(desiredActivations, desiredDerivatives);
|
||||
}
|
||||
|
||||
@@ -4606,6 +4606,106 @@ TEST_CASE("TransposedConvolutionWeightInitializationTest", "[ANNLayerTest]")
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple Test for PixelShuffle layer.
|
||||
*/
|
||||
TEST_CASE("PixelShuffleLayerTest", "[ANNLayerTest]")
|
||||
{
|
||||
arma::mat input1, output1, gy1, g1, outputExpected1, gExpected1;
|
||||
arma::mat input2, output2, gy2, g2, outputExpected2, gExpected2;
|
||||
PixelShuffle<> module1(2, 2, 2, 4);
|
||||
PixelShuffle<> module2(2, 2, 2, 4);
|
||||
|
||||
// Input is a single image, of size (2,2) and having 4 channels.
|
||||
input1 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0
|
||||
<< 0 << 0 << arma::endr;
|
||||
gy1 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8
|
||||
<< 12 << 16 << arma::endr;
|
||||
|
||||
// Calculated using torch.nn.PixelShuffle().
|
||||
outputExpected1 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0
|
||||
<< 0 << 0 << 0 << 0 << arma::endr;
|
||||
gExpected1 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12
|
||||
<< 6 << 14 << 8 << 16 << arma::endr;
|
||||
|
||||
input1 = input1.t();
|
||||
outputExpected1 = outputExpected1.t();
|
||||
gy1 = gy1.t();
|
||||
gExpected1 = gExpected1.t();
|
||||
|
||||
// Check the Forward pass of the layer.
|
||||
module1.Forward(input1, output1);
|
||||
CheckMatrices(output1, outputExpected1);
|
||||
|
||||
// Check the Backward pass of the layer.
|
||||
module1.Backward(input1, gy1, g1);
|
||||
CheckMatrices(g1, gExpected1);
|
||||
|
||||
// Input is a batch of 2 images, each of size (2,2) and having 4 channels.
|
||||
input2 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0
|
||||
<< 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0
|
||||
<< 0 << 0 << 0 << 0 << 0 << 0 << arma::endr;
|
||||
gy2 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8
|
||||
<< 12 << 16 << arma::endr << 17 << 21 << 25 << 29 << 18 << 22 << 26 << 30
|
||||
<< 19 << 23 << 27 << 31 << 20 << 24 << 28 << 32 << arma::endr;
|
||||
|
||||
// Calculated using torch.nn.PixelShuffle().
|
||||
outputExpected2 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0
|
||||
<< 0 << 0 << 0 << 0 << arma::endr << 5 << 0 << 7 << 0 << 0 << 0 << 0 << 0
|
||||
<< 6 << 0 << 8 << 0 << 0 << 0 << 0 << 0 << arma::endr;
|
||||
gExpected2 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12
|
||||
<< 6 << 14 << 8 << 16 << arma::endr << 17 << 25 << 19 << 27 << 21 << 29
|
||||
<< 23 << 31 << 18 << 26 << 20 << 28 << 22 << 30 << 24 << 32 << arma::endr;
|
||||
|
||||
input2 = input2.t();
|
||||
outputExpected2 = outputExpected2.t();
|
||||
gy2 = gy2.t();
|
||||
gExpected2 = gExpected2.t();
|
||||
|
||||
// Check the Forward pass of the layer.
|
||||
module2.Forward(input2, output2);
|
||||
CheckMatrices(output2, outputExpected2);
|
||||
|
||||
// Check the Backward pass of the layer.
|
||||
module2.Backward(input2, gy2, g2);
|
||||
CheckMatrices(g2, gExpected2);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that the function that can access the parameters of the
|
||||
* PixelShuffle layer works.
|
||||
*/
|
||||
TEST_CASE("PixelShuffleLayerParametersTest", "[ANNLayerTest]")
|
||||
{
|
||||
// Create the layer using the empty constructor.
|
||||
PixelShuffle<> layer;
|
||||
|
||||
// Set the different input parameters of the layer.
|
||||
layer.UpscaleFactor() = 2;
|
||||
layer.InputHeight() = 2;
|
||||
layer.InputWidth() = 2;
|
||||
layer.InputChannels() = 4;
|
||||
|
||||
// Make sure we can get the parameters successfully.
|
||||
REQUIRE(layer.UpscaleFactor() == 2);
|
||||
REQUIRE(layer.InputHeight() == 2);
|
||||
REQUIRE(layer.InputWidth() == 2);
|
||||
REQUIRE(layer.InputChannels() == 4);
|
||||
|
||||
arma::mat input, output;
|
||||
// Input is a batch of 2 images, each of size (2,2) and having 4 channels.
|
||||
input << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0
|
||||
<< 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0
|
||||
<< 0 << 0 << 0 << 0 << 0 << 0 << arma::endr;
|
||||
input = input.t();
|
||||
layer.Forward(input, output);
|
||||
|
||||
// Check whether output parameters are returned correctly.
|
||||
REQUIRE(layer.OutputHeight() == 4);
|
||||
REQUIRE(layer.OutputWidth() == 4);
|
||||
REQUIRE(layer.OutputChannels() == 1);
|
||||
}
|
||||
|
||||
/*
|
||||
* Simple Test for SpatialDropout layer.
|
||||
*/
|
||||
TEST_CASE("SpatialDropoutLayerTest", "[ANNLayerTest]")
|
||||
|
||||
Reference in New Issue
Block a user