From 3660b340a109530decbebb50860151d6cfde4f55 Mon Sep 17 00:00:00 2001
From: kartikdutt18 <kartikdutt@live.in>
Date: Thu, 27 Feb 2020 16:49:13 +0530
Subject: [PATCH] Refactor Activation Function Codes

---
 src/mlpack/methods/ann/layer/elu.hpp          | 63 -------------------
 src/mlpack/methods/ann/layer/elu_impl.hpp     | 23 +++++--
 src/mlpack/methods/ann/layer/leaky_relu.hpp   | 52 ---------------
 .../methods/ann/layer/leaky_relu_impl.hpp     |  7 ++-
 .../methods/ann/layer/parametric_relu.hpp     | 54 ----------------
 .../ann/layer/parametric_relu_impl.hpp        | 11 +++-
 6 files changed, 33 insertions(+), 177 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/elu.hpp b/src/mlpack/methods/ann/layer/elu.hpp
index 2dd11e8a78..575b99e4d5 100644
--- a/src/mlpack/methods/ann/layer/elu.hpp
+++ b/src/mlpack/methods/ann/layer/elu.hpp
@@ -173,69 +173,6 @@ class ELU
   void serialize(Archive& ar, const unsigned int /* version */);
 
  private:
-  /**
-   * Computes the value of activation function.
-   *
-   * @param x Input data.
-   * @return f(x).
-   */
-  double Fn(const double x)
-  {
-    if (x < DBL_MAX)
-    {
-      return (x > 0) ? lambda * x : lambda * alpha * (std::exp(x) - 1);
-    }
-
-    return 1.0;
-  }
-
-  /**
-   * Computes the value of activation function using a dense matrix as input.
-   *
-   * @param x Input data.
-   * @param y The resulting output activation.
-   */
-  template<typename eT>
-  void Fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
-  {
-    y.set_size(arma::size(x));
-
-    for (size_t i = 0; i < x.n_elem; i++)
-    {
-      y(i) = Fn(x(i));
-    }
-  }
-
-  /**
-   * Computes the first derivative of the activation function.
-   *
-   * @param x Input data.
-   * @param y Propagated data f(x).
-   * @return f'(x)
-   */
-  double Deriv(const double x, const double y)
-  {
-    return (x > 0) ? lambda : y + lambda * alpha;
-  }
-
-  /**
-   * Computes the first derivative of the activation function.
-   *
-   * @param x Input data.
-   * @param y Output activations f(x).
-   * @param z The resulting derivatives.
-   */
-  template<typename InputType, typename OutputType>
-  void Deriv(const InputType& x, OutputType& y)
-  {
-    derivative.set_size(arma::size(x));
-
-    for (size_t i = 0; i < x.n_elem; i++)
-    {
-      derivative(i) = Deriv(x(i), y(i));
-    }
-  }
-
   //! Locally-stored delta object.
   OutputDataType delta;
 
diff --git a/src/mlpack/methods/ann/layer/elu_impl.hpp b/src/mlpack/methods/ann/layer/elu_impl.hpp
index 2cbbe7f291..3c94871e49 100644
--- a/src/mlpack/methods/ann/layer/elu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/elu_impl.hpp
@@ -51,12 +51,27 @@ template<typename InputType, typename OutputType>
 void ELU<InputDataType, OutputDataType>::Forward(
     const InputType&& input, OutputType&& output)
 {
-  Fn(input, output);
-
-  if (!deterministic)
+  output.set_size(arma::size(input));
+  for (size_t i = 0; i < input.n_elem; i++)
   {
-    Deriv(input, output);
+    if (input(i) < DBL_MAX)
+    {
+      output(i) = (input(i) > 0) ? lambda * input(i) : lambda *
+          alpha * (std::exp(input(i)) - 1);
+    }
+    else
+      output(i) = 1.0;
   }
+
+    if (!deterministic)
+    {
+      derivative.set_size(arma::size(input));
+      for (size_t i = 0; i < input.n_elem; i++)
+      {
+        derivative(i) = (input(i) > 0) ? lambda : output(i) +
+            lambda * alpha;
+      }
+    }
 }
 
 template<typename InputDataType, typename OutputDataType>
diff --git a/src/mlpack/methods/ann/layer/leaky_relu.hpp b/src/mlpack/methods/ann/layer/leaky_relu.hpp
index aec7655f71..f9d7b230f9 100644
--- a/src/mlpack/methods/ann/layer/leaky_relu.hpp
+++ b/src/mlpack/methods/ann/layer/leaky_relu.hpp
@@ -97,58 +97,6 @@ class LeakyReLU
   void serialize(Archive& ar, const unsigned int /* version */);
 
  private:
-  /**
-   * Computes the LeakyReLU function
-   *
-   * @param x Input data.
-   * @return f(x).
-   */
-  double Fn(const double x)
-  {
-    return std::max(x, alpha * x);
-  }
-
-  /**
-   * Computes the LeakyReLU function using a dense matrix as input.
-   *
-   * @param x Input data.
-   * @param y The resulting output activation.
-   */
-  template<typename eT>
-  void Fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
-  {
-    y = arma::max(x, alpha * x);
-  }
-
-  /**
-   * Computes the first derivative of the LeakyReLU function.
-   *
-   * @param x Input data.
-   * @return f'(x)
-   */
-  double Deriv(const double x)
-  {
-    return (x >= 0) ? 1 : alpha;
-  }
-
-  /**
-   * Computes the first derivative of the LeakyReLU function.
-   *
-   * @param x Input activations.
-   * @param y The resulting derivatives.
-   */
-
-  template<typename InputType, typename OutputType>
-  void Deriv(const InputType& x, OutputType& y)
-  {
-    y.set_size(arma::size(x));
-
-    for (size_t i = 0; i < x.n_elem; i++)
-    {
-      y(i) = Deriv(x(i));
-    }
-  }
-
   //! Locally-stored delta object.
   OutputDataType delta;
 
diff --git a/src/mlpack/methods/ann/layer/leaky_relu_impl.hpp b/src/mlpack/methods/ann/layer/leaky_relu_impl.hpp
index 51a2d75113..5d024ad3dd 100644
--- a/src/mlpack/methods/ann/layer/leaky_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/leaky_relu_impl.hpp
@@ -32,7 +32,7 @@ template<typename InputType, typename OutputType>
 void LeakyReLU<InputDataType, OutputDataType>::Forward(
     const InputType&& input, OutputType&& output)
 {
-  Fn(input, output);
+  output = arma::max(input, alpha * input);
 }
 
 template<typename InputDataType, typename OutputDataType>
@@ -41,7 +41,10 @@ void LeakyReLU<InputDataType, OutputDataType>::Backward(
     const DataType&& input, DataType&& gy, DataType&& g)
 {
   DataType derivative;
-  Deriv(input, derivative);
+  derivative.set_size(arma::size(input));
+  for (size_t i = 0; i < input.n_elem; i++)
+    derivative(i) = (input(i) >= 0) ? 1 : alpha;
+
   g = gy % derivative;
 }
 
diff --git a/src/mlpack/methods/ann/layer/parametric_relu.hpp b/src/mlpack/methods/ann/layer/parametric_relu.hpp
index f1bcbbc909..9edde60190 100644
--- a/src/mlpack/methods/ann/layer/parametric_relu.hpp
+++ b/src/mlpack/methods/ann/layer/parametric_relu.hpp
@@ -126,60 +126,6 @@ class PReLU
   void serialize(Archive& ar, const unsigned int /* version */);
 
  private:
-  /**
-   * Computes the parametric ReLU function.
-   *
-   * @param x Input data.
-   * @return f(x).
-   */
-  double Fn(const double x)
-  {
-    return std::max(x, alpha(0) * x);
-  }
-
-  /**
-   * Computes the parametric ReLU function using a dense matrix as input.
-   *
-   * @param x Input data.
-   * @param y The resulting output activation.
-   */
-  template<typename eT>
-  void Fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
-  {
-    y = x;
-    arma::uvec negative = arma::find(x < 0);
-    y(negative) = x(negative) * alpha(0);
-  }
-
-  /**
-   * Computes the first derivative of the parametric ReLU function.
-   *
-   * @param x Input data.
-   * @return f'(x)
-   */
-  double Deriv(const double x)
-  {
-    return (x >= 0) ? 1 : alpha(0);
-  }
-
-  /**
-   * Computes the first derivative of the PReLU function.
-   *
-   * @param x Input activations.
-   * @param y The resulting derivatives.
-   */
-
-  template<typename InputType, typename OutputType>
-  void Deriv(const InputType& x, OutputType& y)
-  {
-    y.set_size(arma::size(x));
-
-    for (size_t i = 0; i < x.n_elem; i++)
-    {
-      y(i) = Deriv(x(i));
-    }
-  }
-
   //! Locally-stored delta object.
   OutputDataType delta;
 
diff --git a/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp
index 67bcb7e378..d7c8ac4ffb 100644
--- a/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp
@@ -41,7 +41,9 @@ template<typename InputType, typename OutputType>
 void PReLU<InputDataType, OutputDataType>::Forward(
     const InputType&& input, OutputType&& output)
 {
-  Fn(input, output);
+  output = input;
+  arma::uvec negative = arma::find(input < 0);
+  output(negative) = input(negative) * alpha(0);
 }
 
 template<typename InputDataType, typename OutputDataType>
@@ -50,7 +52,12 @@ void PReLU<InputDataType, OutputDataType>::Backward(
     const DataType&& input, DataType&& gy, DataType&& g)
 {
   DataType derivative;
-  Deriv(input, derivative);
+  derivative.set_size(arma::size(input));
+  for (size_t i = 0; i < input.n_elem; i++)
+  {
+    derivative(i) = (input(i) >= 0) ? 1 : alpha(0);
+  }
+
   g = gy % derivative;
 }