diff --git a/HISTORY.md b/HISTORY.md
index 0557b30e1e..e7e21829f5 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -2,6 +2,8 @@
 ###### ????-??-??
   * Added `Multi Label Soft Margin Loss` loss function for neural networks
    (#2345).
+   
+  * Added Pixel Shuffle layer (#2563).
 
   * Add "check_input_matrices" option to python bindings that checks
     for NaN and inf values in all the input matrices (#2787).
@@ -55,7 +57,6 @@
 
 ### mlpack 3.4.0
 ###### 2020-09-01
-
   * Issue warnings when metrics produce NaNs in KFoldCV (#2595).
 
   * Added bindings for _R_ during Google Summer of Code (#2556).
diff --git a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt
index fd4e765006..d5c0868c1c 100644
--- a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt
+++ b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt
@@ -19,6 +19,7 @@ set(SOURCES
   multi_quadratic_function.hpp
   poisson1_function.hpp
   gaussian_function.hpp
+  hard_swish_function.hpp
 )
 
 # Add directory name to sources.
diff --git a/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp b/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp
new file mode 100644
index 0000000000..d387e86474
--- /dev/null
+++ b/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp
@@ -0,0 +1,116 @@
+/**
+ * @file methods/ann/activation_functions/hard_swish_function.hpp
+ * @author Anush Kini
+ *
+ * Definition and implementation of the Hard Swish function as described by
+ * Howard A, Sandler M, Chu G, Chen LC, Chen B, Tan M, Wang W, Zhu Y, Pang R,
+ * Vasudevan V and Le QV.
+ * For more information, see the following paper.
+ *
+ * @code
+ * @misc{
+ *   author = {Howard A, Sandler M, Chu G, Chen LC, Chen B, Tan M, Wang W,
+ *            Zhu Y, Pang R, Vasudevan V and Le QV},
+ *   title = {Searching for MobileNetV3},
+ *   year = {2019}
+ * }
+ * @endcode
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_HARD_SWISH_FUNCTION_HPP
+#define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_HARD_SWISH_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+/**
+ * The Hard Swish function, defined by
+ *
+ * @f{eqnarray*}{
+ *   f(x) &=& \begin{cases}
+ *     0 & x \leq -3\\
+ *     x & x \geq +3\\
+ *     \frac{x * (x + 3)}{6} & otherwise\\
+ *   \end{cases} \\
+ *   f'(x) &=& \begin{cases}
+ *     0 & x \leq -3\\
+ *     1 & x \geq +3\\
+ *     \frac{2x + 3}{6} & otherwise\\
+ *   \end{cases}
+ * @f}
+ */
+class HardSwishFunction
+{
+ public:
+  /**
+   * Computes the Hard Swish function.
+   *
+   * @param x Input data.
+   * @return f(x).
+   */
+  static double Fn(const double x)
+  {
+    if (x <= -3)
+      return 0;
+    else if (x >= 3)
+      return x;
+
+    return x * (x + 3) / 6;
+  }
+
+  /**
+   * Computes the Hard Swish function.
+   *
+   * @param x Input data.
+   * @param y The resulting output activation.
+   */
+  template <typename InputVecType, typename OutputVecType>
+  static void Fn(const InputVecType &x, OutputVecType &y)
+  {
+    y.set_size(size(x));
+
+    for (size_t i = 0; i < x.n_elem; i++)
+      y(i) = Fn(x(i));
+  }
+
+  /**
+   * Computes the first derivative of the Hard Swish function.
+   *
+   * @param y Input data.
+   * @return f'(x).
+   */
+  static double Deriv(const double y)
+  {
+    if (y <= -3)
+      return 0;
+    else if (y >= 3)
+      return 1;
+
+    return (2 * y + 3.0) / 6.0;
+  }
+
+  /**
+   * Computes the first derivatives of the Hard Swish function.
+   *
+   * @param y Input data.
+   * @param x The resulting derivatives.
+   */
+  template <typename InputVecType, typename OutputVecType>
+  static void Deriv(const InputVecType &y, OutputVecType &x)
+  {
+    x.set_size(size(y));
+
+    for (size_t i = 0; i < y.n_elem; i++)
+      x(i) = Deriv(y(i));
+  }
+}; // class HardSwishFunction
+
+} // namespace ann
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt
index 5fe560edd4..2b181012c7 100644
--- a/src/mlpack/methods/ann/layer/CMakeLists.txt
+++ b/src/mlpack/methods/ann/layer/CMakeLists.txt
@@ -46,6 +46,8 @@ set(SOURCES
   hard_tanh_impl.hpp
   highway.hpp
   highway_impl.hpp
+  isrlu.hpp
+  isrlu_impl.hpp
   join.hpp
   join_impl.hpp
   layer.hpp
@@ -83,6 +85,8 @@ set(SOURCES
   noisylinear_impl.hpp
   parametric_relu.hpp
   parametric_relu_impl.hpp
+  pixel_shuffle.hpp
+  pixel_shuffle_impl.hpp
   positional_encoding.hpp
   positional_encoding_impl.hpp
   recurrent.hpp
diff --git a/src/mlpack/methods/ann/layer/base_layer.hpp b/src/mlpack/methods/ann/layer/base_layer.hpp
index 8429c818a7..ae49f30fe6 100644
--- a/src/mlpack/methods/ann/layer/base_layer.hpp
+++ b/src/mlpack/methods/ann/layer/base_layer.hpp
@@ -27,6 +27,7 @@
 #include <mlpack/methods/ann/activation_functions/elliot_function.hpp>
 #include <mlpack/methods/ann/activation_functions/elish_function.hpp>
 #include <mlpack/methods/ann/activation_functions/gaussian_function.hpp>
+#include <mlpack/methods/ann/activation_functions/hard_swish_function.hpp>
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
@@ -50,6 +51,7 @@ namespace ann /** Artificial Neural Network. */ {
  *  - ELiSHLayer
  *  - ElliotLayer
  *  - GaussianLayer
+ *  - HardSwishLayer
  *
  * @tparam ActivationFunction Activation function used for the embedding layer.
  * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
@@ -277,6 +279,17 @@ template <
 using GaussianFunctionLayer = BaseLayer<
     ActivationFunction, InputDataType, OutputDataType>;
 
+/**
+ * Standard HardSwish-Layer using the HardSwish activation function.
+ */
+template <
+    class ActivationFunction = HardSwishFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using HardSwishFunctionLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
+
 } // namespace ann
 } // namespace mlpack
 
diff --git a/src/mlpack/methods/ann/layer/isrlu.hpp b/src/mlpack/methods/ann/layer/isrlu.hpp
new file mode 100644
index 0000000000..b0a786c6ba
--- /dev/null
+++ b/src/mlpack/methods/ann/layer/isrlu.hpp
@@ -0,0 +1,138 @@
+/**
+ * @file methods/ann/layer/isrlu.hpp
+ * @author Abhinav Anand
+ *
+ * Definition of the ISRLU activation function as described by Jonathan T. Barron.
+ *
+ * For more information, read the following paper.
+ *
+ * @code
+ * @article{
+ *   author  = {Carlile, Brad and Delamarter, Guy and Kinney, Paul and Marti,
+ *              Akiko and Whitney, Brian},
+ *   title   = {Improving deep learning by inverse square root linear units (ISRLUs)},
+ *   year    = {2017},
+ *   url     = {https://arxiv.org/pdf/1710.09967.pdf}
+ * }
+ * @endcode
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_ISRLU_HPP
+#define MLPACK_METHODS_ANN_LAYER_ISRLU_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * The ISRLU activation function, defined by
+ *
+ * @f{eqnarray*}{
+ * f(x) &=& \left\{
+ *   \begin{array}{lr}
+ *    x & : x \ge 0 \\
+ *    x(\frac{1}{1 + \alpha x^2}) & : x < 0
+ *   \end{array}
+ * \right. \\
+ * f'(x) &=& \left\{
+ *   \begin{array}{lr}
+ *    x & : 1 \ge 0 \\
+ *    (\frac{1}{1 + \alpha x^2})^3 & : x < 0
+ *   \end{array}
+ * \right. 
+ * @f}
+ *
+ *
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ */
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+class ISRLU
+{
+ public:
+  /**
+   * Create the ISRLU object using the specified parameter.
+   *
+   * @param alpha Scale parameter controls the value to which an ISRLU
+   *        saturates for negative inputs.
+   */
+  ISRLU(const double alpha = 1.0);
+
+  /**
+   * Ordinary feed forward pass of a neural network, evaluating the function
+   * f(x) by propagating the activity forward through f.
+   *
+   * @param input Input data used for evaluating the specified function.
+   * @param output Resulting output activation.
+   */
+  template<typename InputType, typename OutputType>
+  void Forward(const InputType& input, OutputType& output);
+
+  /**
+   * Ordinary feed backward pass of a neural network, calculating the function
+   * f(x) by propagating x backwards through f. Using the results from the feed
+   * forward pass.
+   *
+   * @param input The propagated input activation f(x).
+   * @param gy The backpropagated error.
+   * @param g The calculated gradient.
+   */
+  template<typename DataType>
+  void Backward(const DataType& input, const DataType& gy, DataType& g);
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  //! Get the non zero gradient.
+  double const& Alpha() const { return alpha; }
+  //! Modify the non zero gradient.
+  double& Alpha() { return alpha; }
+
+  //! Get size of weights.
+  size_t WeightSize() { return 0; }
+
+  /**
+   * Serialize the layer.
+   */
+  template<typename Archive>
+  void serialize(Archive& ar, const uint32_t /* version */);
+
+ private:
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
+  //! Locally stored first derivative of the activation function.
+  arma::mat derivative;
+
+  //! ISRLU Hyperparameter (alpha > 0).
+  double alpha;
+
+}; // class ISRLU
+
+} // namespace ann
+} // namespace mlpack
+
+// Include implementation.
+#include "isrlu_impl.hpp"
+
+#endif
diff --git a/src/mlpack/methods/ann/layer/isrlu_impl.hpp b/src/mlpack/methods/ann/layer/isrlu_impl.hpp
new file mode 100644
index 0000000000..a91830f51d
--- /dev/null
+++ b/src/mlpack/methods/ann/layer/isrlu_impl.hpp
@@ -0,0 +1,65 @@
+/**
+ * @file methods/ann/layer/isrlu_impl.hpp
+ * @author Abhinav Anand
+ *
+ * Implementation of the ISRLU activation function as described by Jonathan T. Barron.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_ISRLU_IMPL_HPP
+#define MLPACK_METHODS_ANN_LAYER_ISRLU_IMPL_HPP
+
+// In case it hasn't yet been included.
+#include "isrlu.hpp"
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+template<typename InputDataType, typename OutputDataType>
+ISRLU<InputDataType, OutputDataType>::ISRLU(const double alpha) :
+    alpha(alpha)
+{}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename InputType, typename OutputType>
+void ISRLU<InputDataType, OutputDataType>::Forward(
+    const InputType& input, OutputType& output)
+{
+  output = arma::ones<OutputDataType>(arma::size(input));
+  for (size_t i = 0; i < input.n_elem; ++i)
+  {
+    output(i) = (input(i) >= 0) ? input(i) : input(i) *
+        (1 / std::sqrt(1 + alpha * (input(i) * input(i))));
+  }
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename DataType>
+void ISRLU<InputDataType, OutputDataType>::Backward(
+    const DataType& input, const DataType& gy, DataType& g)
+{
+  derivative.set_size(arma::size(input));
+  for (size_t i = 0; i < input.n_elem; ++i)
+  {
+    derivative(i) = (input(i) >= 0) ? 1 :
+        std::pow(1 / std::sqrt(1 + alpha * input(i) * input(i)), 3);
+  }
+  g = gy % derivative;
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename Archive>
+void ISRLU<InputDataType, OutputDataType>::serialize(
+    Archive& ar,
+    const uint32_t /* version */)
+{
+  ar(CEREAL_NVP(alpha));
+}
+
+} // namespace ann
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/methods/ann/layer/layer.hpp b/src/mlpack/methods/ann/layer/layer.hpp
index 9cf806b7e4..6d13a26772 100644
--- a/src/mlpack/methods/ann/layer/layer.hpp
+++ b/src/mlpack/methods/ann/layer/layer.hpp
@@ -58,6 +58,7 @@
 #include "noisylinear.hpp"
 #include "padding.hpp"
 #include "parametric_relu.hpp"
+#include "pixel_shuffle.hpp"
 #include "positional_encoding.hpp"
 #include "recurrent_attention.hpp"
 #include "recurrent.hpp"
diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp
index 091d7ece35..2532efecfe 100644
--- a/src/mlpack/methods/ann/layer/layer_types.hpp
+++ b/src/mlpack/methods/ann/layer/layer_types.hpp
@@ -43,6 +43,7 @@
 #include <mlpack/methods/ann/layer/adaptive_max_pooling.hpp>
 #include <mlpack/methods/ann/layer/adaptive_mean_pooling.hpp>
 #include <mlpack/methods/ann/layer/parametric_relu.hpp>
+#include <mlpack/methods/ann/layer/pixel_shuffle.hpp>
 #include <mlpack/methods/ann/layer/positional_encoding.hpp>
 #include <mlpack/methods/ann/layer/reinforce_normal.hpp>
 #include <mlpack/methods/ann/layer/reparametrization.hpp>
@@ -53,6 +54,7 @@
 #include <mlpack/methods/ann/layer/virtual_batch_norm.hpp>
 #include <mlpack/methods/ann/layer/hardshrink.hpp>
 #include <mlpack/methods/ann/layer/celu.hpp>
+#include <mlpack/methods/ann/layer/isrlu.hpp>
 #include <mlpack/methods/ann/layer/softshrink.hpp>
 #include <mlpack/methods/ann/layer/radial_basis_function.hpp>
 
@@ -221,6 +223,7 @@ class AdaptiveMeanPooling;
 using MoreTypes = boost::variant<
         Linear3D<arma::mat, arma::mat, NoRegularizer>*,
         LpPooling<arma::mat, arma::mat>*,
+        PixelShuffle<arma::mat, arma::mat>*,
         Glimpse<arma::mat, arma::mat>*,
         Highway<arma::mat, arma::mat>*,
         MultiheadAttention<arma::mat, arma::mat, NoRegularizer>*,
@@ -236,7 +239,8 @@ using MoreTypes = boost::variant<
         VirtualBatchNorm<arma::mat, arma::mat>*,
         RBF<arma::mat, arma::mat, GaussianFunction>*,
         BaseLayer<GaussianFunction, arma::mat, arma::mat>*,
-        PositionalEncoding<arma::mat, arma::mat>*
+        PositionalEncoding<arma::mat, arma::mat>*,
+        ISRLU<arma::mat, arma::mat>*
 >;
 
 template <typename... CustomLayers>
diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle.hpp
new file mode 100644
index 0000000000..f425d7c508
--- /dev/null
+++ b/src/mlpack/methods/ann/layer/pixel_shuffle.hpp
@@ -0,0 +1,181 @@
+/**
+ * @file methods/ann/layer/pixel_shuffle.hpp
+ * @author Anjishnu Mukherjee
+ * @author Abhinav Anand
+ *
+ * Definition of the PixelShuffle class.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_HPP
+#define MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * Implementation of the PixelShuffle layer.
+ *
+ * For more information, refer to the following paper,
+ *
+ * @code
+ * @article{Shi16,
+ *   author    = {Wenzhe Shi, Jose Caballero,Ferenc Huszár, Johannes Totz,
+ *               Andrew P. Aitken, Rob Bishop, Daniel Rueckert, Zehan Wang},
+ *   title     = {Real-Time Single Image and Video Super-Resolution Using an
+ *               Efficient Sub-Pixel Convolutional Neural Network},
+ *   journal   = {CoRR},
+ *   volume    = {abs/1609.05158},
+ *   year      = {2016},
+ *   url       = {https://arxiv.org/abs/1609.05158},
+ *   eprint    = {1609.05158},
+ * }
+ * @endcode
+ *
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ */
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+class PixelShuffle
+{
+ public:
+  //! Create the PixelShuffle object.
+  PixelShuffle();
+
+  /**
+   * Create the PixelShuffle object using the specified parameters.
+   * The number of input channels should be an integral multiple of the square
+   * of the upscale factor.
+   *
+   * @param upscaleFactor The scaling factor for Pixel Shuffle.
+   * @param height The height of each input image.
+   * @param width The width of each input image.
+   * @param size The number of channels of each input image.
+   */
+  PixelShuffle(const size_t upscaleFactor,
+               const size_t height,
+               const size_t width,
+               const size_t size);
+
+  /**
+   * Ordinary feed forward pass of the PixelShuffle layer.
+   *
+   * @param input Input data used for evaluating the specified function.
+   * @param output Resulting output activation.
+   */
+  template<typename eT>
+  void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);
+
+  /**
+   * Ordinary feed backward pass of the PixelShuffle layer.
+   *
+   * @param input The propagated input activation.
+   * @param gy The backpropagated error.
+   * @param g The calculated gradient.
+   */
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& input,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g);
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  //! Get the upscale factor.
+  size_t UpscaleFactor() const { return upscaleFactor; }
+
+  //! Modify the upscale factor.
+  size_t& UpscaleFactor() { return upscaleFactor; }
+
+  //! Get the input image height.
+  size_t InputHeight() const { return height; }
+
+  //! Modify the input image height.
+  size_t& InputHeight() { return height; }
+
+  //! Get the input image width.
+  size_t InputWidth() const { return width; }
+
+  //! Modify the input image width.
+  size_t& InputWidth() { return width; }
+
+  //! Get the number of input channels.
+  size_t InputChannels() const { return size; }
+
+  //! Modify the number of input channels.
+  size_t& InputChannels() { return size; }
+
+  //! Get the output image height.
+  size_t OutputHeight() const { return outputHeight; }
+
+  //! Get the output image width.
+  size_t OutputWidth() const { return outputWidth; }
+
+  //! Get the number of output channels.
+  size_t OutputChannels() const { return sizeOut; }
+
+  /**
+   * Serialize the layer.
+   */
+  template<typename Archive>
+  void serialize(Archive& ar, const unsigned int /* version */);
+
+ private:
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
+  //! The scaling factor for Pixel Shuffle.
+  size_t upscaleFactor;
+
+  //! The height of each input image.
+  size_t height;
+
+  //! The width of each input image.
+  size_t width;
+
+  //! The number of channels of each input image.
+  size_t size;
+
+  //! The number of images in the batch.
+  size_t batchSize;
+
+  //! The height of each output image.
+  size_t outputHeight;
+
+  //! The width of each output image.
+  size_t outputWidth;
+
+  //! The number of channels of each output image.
+  size_t sizeOut;
+
+  //! A boolean used to do some internal calculations once initially.
+  bool reset;
+}; // class PixelShuffle
+
+} // namespace ann
+} // namespace mlpack
+
+// Include implementation.
+#include "pixel_shuffle_impl.hpp"
+
+#endif
diff --git a/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp
new file mode 100644
index 0000000000..f56f708981
--- /dev/null
+++ b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp
@@ -0,0 +1,143 @@
+/**
+ * @file methods/ann/layer/pixel_shuffle_impl.hpp
+ * @author Anjishnu Mukherjee
+ * @author Abhinav Anand
+ *
+ * Implementation of the PixelShuffle class.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_IMPL_HPP
+#define MLPACK_METHODS_ANN_LAYER_PIXEL_SHUFFLE_IMPL_HPP
+
+// In case it hasn't yet been included.
+#include "pixel_shuffle.hpp"
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+template<typename InputDataType, typename OutputDataType>
+PixelShuffle<InputDataType, OutputDataType>::PixelShuffle() :
+    PixelShuffle(0, 0, 0, 0)
+{
+  // Nothing to do here.
+}
+
+template<typename InputDataType, typename OutputDataType>
+PixelShuffle<InputDataType, OutputDataType>::PixelShuffle(
+    const size_t upscaleFactor,
+    const size_t height,
+    const size_t width,
+    const size_t size) :
+    upscaleFactor(upscaleFactor),
+    height(height),
+    width(width),
+    size(size),
+    batchSize(0),
+    outputHeight(0),
+    outputWidth(0),
+    sizeOut(0),
+    reset(false)
+{
+  // Nothing to do here.
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void PixelShuffle<InputDataType, OutputDataType>::Forward(
+  const arma::Mat<eT>& input, arma::Mat<eT>& output)
+{
+  if (!reset)
+  {
+    batchSize = input.n_cols;
+    sizeOut = size / std::pow(upscaleFactor, 2);
+    outputHeight = height * upscaleFactor;
+    outputWidth = width * upscaleFactor;
+    reset = true;
+  }
+
+  output.zeros(outputHeight * outputWidth * sizeOut, batchSize);
+  for (size_t n = 0; n < batchSize; n++)
+  {
+    arma::cube inputTemp(const_cast<arma::mat&>(input).memptr(), height,
+        width, size * batchSize, false, false);
+    arma::cube outputTemp(const_cast<arma::mat&>(output).memptr(),
+        outputHeight, outputWidth, sizeOut * batchSize, false, false);
+
+    for (size_t c = 0; c < sizeOut; c++)
+    {
+      for (size_t h = 0; h < outputHeight; h++)
+      {
+        for (size_t w = 0; w < outputWidth; w++)
+        {
+          size_t height_index = h / upscaleFactor;
+          size_t width_index = w / upscaleFactor;
+          size_t channel_index = (upscaleFactor * (h % upscaleFactor)) +
+              (w % upscaleFactor) + (c * std::pow(upscaleFactor, 2));
+          outputTemp(w, h, c + n * sizeOut) = inputTemp(width_index, height_index,
+              channel_index + n * size);
+        }
+      }
+    }
+
+  }
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void PixelShuffle<InputDataType, OutputDataType>::Backward(
+    const arma::Mat<eT>& input, const arma::Mat<eT>& gy, arma::Mat<eT>& g)
+{
+  g.zeros(arma::size(input));
+  for (size_t n = 0; n < batchSize; n++)
+  {
+    arma::cube gyTemp(const_cast<arma::mat&>(gy).memptr(), outputHeight,
+        outputWidth, sizeOut * batchSize, false, false);
+    arma::cube gTemp(const_cast<arma::mat&>(g).memptr(), height, width,
+        size * batchSize, false, false);
+
+    for (size_t c = 0; c < sizeOut; c++)
+    {
+      for (size_t h = 0; h < outputHeight; h++)
+      {
+        for (size_t w = 0; w < outputWidth; w++)
+        {
+          size_t height_index = h / upscaleFactor;
+          size_t width_index = w / upscaleFactor;
+          size_t channel_index = (upscaleFactor * (h % upscaleFactor)) +
+              (w % upscaleFactor) + (c * std::pow(upscaleFactor, 2));
+          gTemp(width_index, height_index, channel_index + n * size) = gyTemp(w, h,
+              c + n * sizeOut);
+        }
+      }
+    }
+
+  }
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename Archive>
+void PixelShuffle<InputDataType, OutputDataType>::serialize(
+    Archive& ar,
+    const unsigned int /* version */)
+{
+  ar(CEREAL_NVP(delta));
+  ar(CEREAL_NVP(outputParameter));
+  ar(CEREAL_NVP(upscaleFactor));
+  ar(CEREAL_NVP(height));
+  ar(CEREAL_NVP(width));
+  ar(CEREAL_NVP(size));
+  ar(CEREAL_NVP(batchSize));
+  ar(CEREAL_NVP(outputHeight));
+  ar(CEREAL_NVP(outputHeight));
+  ar(CEREAL_NVP(outputWidth));
+  ar(CEREAL_NVP(sizeOut));
+}
+
+} // namespace ann
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt
index e6dd629266..e94c73e0d3 100644
--- a/src/mlpack/tests/CMakeLists.txt
+++ b/src/mlpack/tests/CMakeLists.txt
@@ -34,6 +34,7 @@ add_executable(mlpack_test
   facilities_test.cpp
   fastmks_test.cpp
   feedforward_network_test.cpp
+  feedforward_network_2_test.cpp
   gan_test.cpp
   gmm_test.cpp
   hmm_test.cpp
diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp
index d88c0a5109..85cc1f7608 100644
--- a/src/mlpack/tests/activation_functions_test.cpp
+++ b/src/mlpack/tests/activation_functions_test.cpp
@@ -32,6 +32,7 @@
 #include <mlpack/methods/ann/activation_functions/spline_function.hpp>
 #include <mlpack/methods/ann/activation_functions/poisson1_function.hpp>
 #include <mlpack/methods/ann/activation_functions/gaussian_function.hpp>
+#include <mlpack/methods/ann/activation_functions/hard_swish_function.hpp>
 
 #include "catch.hpp"
 
@@ -558,6 +559,54 @@ void CheckCELUDerivativeCorrect(const arma::colvec input,
   }
 }
 
+/**
+ * Implementation of the ISRLU activation function test. The function is
+ * implemented as ISRLU layer in the file isrlu.hpp.
+ *
+ * @param input Input data used for evaluating the ISRLU activation function.
+ * @param target Target data used to evaluate the ISRLU activation.
+ */
+void CheckISRLUActivationCorrect(const arma::colvec input,
+                                 const arma::colvec target)
+{
+  // Initialize ISRLU object with alpha = 1.0.
+  ISRLU<> lrf(1.0);
+
+  // Test the activation function using the entire vector as input.
+  arma::colvec activations;
+  lrf.Forward(input, activations);
+  for (size_t i = 0; i < activations.n_elem; ++i)
+  {
+    REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5));
+  }
+}
+
+/**
+ * Implementation of the ISRLU activation function derivative test. The function
+ * is implemented as ISRLU layer in the file isrlu.hpp.
+ *
+ * @param input Input data used for evaluating the ISRLU activation function.
+ * @param target Target data used to evaluate the ISRLU activation.
+ */
+void CheckISRLUDerivativeCorrect(const arma::colvec input,
+                                 const arma::colvec target)
+{
+  // Initialize ISRLU object with alpha = 1.0.
+  ISRLU<> lrf(1.0);
+
+  // Test the calculation of the derivatives using the entire vector as input.
+  arma::colvec derivatives, activations;
+
+  // This error vector will be set to 1 to get the derivatives.
+  arma::colvec error = arma::ones<arma::colvec>(input.n_elem);
+  lrf.Forward(input, activations);
+  lrf.Backward(activations, error, derivatives);
+  for (size_t i = 0; i < derivatives.n_elem; ++i)
+  {
+    REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5));
+  }
+}
+
 /**
  * Implementation of the Softmin activation function test. The function is
  * implemented as Softmin layer in the file softmin.hpp.
@@ -991,6 +1040,22 @@ TEST_CASE("CELUFunctionTest", "[ActivationFunctionsTest]")
   CheckCELUDerivativeCorrect(desiredActivations, desiredDerivatives);
 }
 
+/**
+ * Basic test of the ISRLU activation function.
+ */
+TEST_CASE("ISRLUFunctionTest", "[ActivationFunctionsTest]")
+{
+  const arma::colvec desiredActivations("-0.89442719 3.2 4.5 \
+                                         -0.99995020 1 -0.70710678 2 0");
+
+  const arma::colvec desiredDerivatives("0.41408666 1 1 \
+                                         0.35357980 1 \
+                                         0.54433105 1 1");
+
+  CheckISRLUActivationCorrect(activationData, desiredActivations);
+  CheckISRLUDerivativeCorrect(activationData, desiredDerivatives);
+}
+
 /**
  * Basic test of the inverse quadratic function.
  */
@@ -1134,3 +1199,24 @@ TEST_CASE("SoftminFunctionTest", "[ActivationFunctionsTest]")
   CheckSoftminDerivativeCorrect(activationData,
                                 desiredDerivatives);
 }
+
+/**
+ * Basic test of the Hard Swish function.
+ */
+TEST_CASE("HardSwishFunctionTest", "[ActivationFunctionsTest]")
+{
+  // Randomly generated data.
+  const arma::colvec activationData("3.6544 -1.9714 -5.2277 1.5448 2.1164");
+
+  // Hand-calculated values.
+  const arma::colvec desiredActivations("3.6544 -0.3379636 0.0 \
+                                         1.1701345 1.8047248");
+
+  // Hand-calculated values.
+  const arma::colvec desiredDerivatives("1.0 0.38734546 0.5 \
+                                         0.89004483 1.1015749");
+
+  CheckActivationCorrect<HardSwishFunction>(activationData, desiredActivations);
+  CheckDerivativeCorrect<HardSwishFunction>
+      (desiredActivations, desiredDerivatives);
+}
diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp
index dbb5266798..54105b1f9e 100644
--- a/src/mlpack/tests/ann_layer_test.cpp
+++ b/src/mlpack/tests/ann_layer_test.cpp
@@ -4606,6 +4606,106 @@ TEST_CASE("TransposedConvolutionWeightInitializationTest", "[ANNLayerTest]")
 }
 
 /**
+ * Simple Test for PixelShuffle layer.
+ */
+TEST_CASE("PixelShuffleLayerTest", "[ANNLayerTest]")
+{
+  arma::mat input1, output1, gy1, g1, outputExpected1, gExpected1;
+  arma::mat input2, output2, gy2, g2, outputExpected2, gExpected2;
+  PixelShuffle<> module1(2, 2, 2, 4);
+  PixelShuffle<> module2(2, 2, 2, 4);
+
+  // Input is a single image, of size (2,2) and having 4 channels.
+  input1 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0
+      << 0 << 0 << arma::endr;
+  gy1 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8
+      << 12 << 16 << arma::endr;
+
+  // Calculated using torch.nn.PixelShuffle().
+  outputExpected1 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0
+      << 0 << 0 << 0 << 0 << arma::endr;
+  gExpected1 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12
+      << 6 << 14 << 8 << 16 << arma::endr;
+
+  input1 = input1.t();
+  outputExpected1 = outputExpected1.t();
+  gy1 = gy1.t();
+  gExpected1 = gExpected1.t();
+
+  // Check the Forward pass of the layer.
+  module1.Forward(input1, output1);
+  CheckMatrices(output1, outputExpected1);
+
+  // Check the Backward pass of the layer.
+  module1.Backward(input1, gy1, g1);
+  CheckMatrices(g1, gExpected1);
+
+  // Input is a batch of 2 images, each of size (2,2) and having 4 channels.
+  input2 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0
+      << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0
+      << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr;
+  gy2 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8
+      << 12 << 16 << arma::endr << 17 << 21 << 25 << 29 << 18 << 22 << 26 << 30
+      << 19 << 23 << 27 << 31 << 20 << 24 << 28 << 32 << arma::endr;
+
+  // Calculated using torch.nn.PixelShuffle().
+  outputExpected2 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0
+      << 0 << 0 << 0 << 0 << arma::endr << 5 << 0 << 7 << 0 << 0 << 0 << 0 << 0
+      << 6 << 0 << 8 << 0 << 0 << 0 << 0 << 0 << arma::endr;
+  gExpected2 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12
+      << 6 << 14 << 8 << 16 << arma::endr << 17 << 25 << 19 << 27 << 21 << 29
+      << 23 << 31 << 18 << 26 << 20 << 28 << 22 << 30 << 24 << 32 << arma::endr;
+
+  input2 = input2.t();
+  outputExpected2 = outputExpected2.t();
+  gy2 = gy2.t();
+  gExpected2 = gExpected2.t();
+
+  // Check the Forward pass of the layer.
+  module2.Forward(input2, output2);
+  CheckMatrices(output2, outputExpected2);
+
+  // Check the Backward pass of the layer.
+  module2.Backward(input2, gy2, g2);
+  CheckMatrices(g2, gExpected2);
+}
+
+/**
+ * Test that the function that can access the parameters of the
+ * PixelShuffle layer works.
+ */
+TEST_CASE("PixelShuffleLayerParametersTest", "[ANNLayerTest]")
+{
+  // Create the layer using the empty constructor.
+  PixelShuffle<> layer;
+
+  // Set the different input parameters of the layer.
+  layer.UpscaleFactor() = 2;
+  layer.InputHeight() = 2;
+  layer.InputWidth() = 2;
+  layer.InputChannels() = 4;
+
+  // Make sure we can get the parameters successfully.
+  REQUIRE(layer.UpscaleFactor() == 2);
+  REQUIRE(layer.InputHeight() == 2);
+  REQUIRE(layer.InputWidth() == 2);
+  REQUIRE(layer.InputChannels() == 4);
+
+  arma::mat input, output;
+  // Input is a batch of 2 images, each of size (2,2) and having 4 channels.
+  input << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0
+      << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0
+      << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr;
+  input = input.t();
+  layer.Forward(input, output);
+
+  // Check whether output parameters are returned correctly.
+  REQUIRE(layer.OutputHeight() == 4);
+  REQUIRE(layer.OutputWidth() == 4);
+  REQUIRE(layer.OutputChannels() == 1);
+}
+
+/*
  * Simple Test for SpatialDropout layer.
  */
 TEST_CASE("SpatialDropoutLayerTest", "[ANNLayerTest]")