Adding changes in RBF and adding activation function

src/mlpack/methods/ann/activation_functions/CMakeLists.txt

@@ -18,6 +18,7 @@ set(SOURCES
   spline_function.hpp
   multi_quadratic_function.hpp
   poisson1_function.hpp
+  gaussian_function.hpp
 )
 
 # Add directory name to sources.

src/mlpack/methods/ann/activation_functions/gaussian_function.hpp

@@ -0,0 +1,83 @@
+/**
+ * @file gaussian_function.hpp
+ * @author Himanshu Pathak
+ *
+ * Definition and implementation of the logistic function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_GAUSSIAN_FUNCTION_HPP
+#define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_GAUSSIAN_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * The logistic function, defined by
+ *
+ * @f{eqnarray*}{
+ * f(x) &=& \frac{1}{1 + e^{-x}} \\
+ * f'(x) &=& f(x) * (1 - f(x)) \\
+ * f^{-1}(y) &=& ln(\frac{y}{1-y})
+ * @f}
+ */
+class GaussianFunction
+{
+ public:
+  /**
+   * Computes the gaussian function.
+   *
+   * @param x Input data.
+   * @return f(x).
+   */
+  template<typename eT>
+  static double Fn(const eT x)
+  {
+    return std::exp(-1 * std::pow(x, 2));
+  }
+
+  /**
+   * Computes the gaussian function.
+   *
+   * @param x Input data.
+   * @param y The resulting output activation.
+   */
+  template<typename InputVecType, typename OutputVecType>
+  static void Fn(const InputVecType& x, OutputVecType& y)
+  {
+    y = arma::exp(-1 * arma::pow(x, 2));
+  }
+
+  /**
+   * Computes the first derivative of the logistic function.
+   *
+   * @param x Input data.
+   * @return f'(x)
+   */
+  static double Deriv(const double y)
+  {
+    return 2 * -y * std::exp(-1 * std::pow(y, 2));
+  }
+
+  /**
+   * Computes the first derivatives of the logistic function.
+   *
+   * @param y Input activations.
+   * @param x The resulting derivatives.
+   */
+  template<typename InputVecType, typename OutputVecType>
+  static void Deriv(const InputVecType& y, OutputVecType& x)
+  {
+    x = 2 * -y % arma::exp(-1 * arma::pow(y, 2));
+  }
+}; // class LogisticFunction
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/layer/CMakeLists.txt

@@ -87,6 +87,8 @@ set(SOURCES
   reinforce_normal_impl.hpp
   reparametrization.hpp
   reparametrization_impl.hpp
+  radial_basis_function.hpp
+  radial_basis_function_impl.hpp
   select.hpp
   select_impl.hpp
   sequential.hpp

src/mlpack/methods/ann/layer/base_layer.hpp

@@ -26,6 +26,7 @@
 #include <mlpack/methods/ann/activation_functions/gelu_function.hpp>
 #include <mlpack/methods/ann/activation_functions/elliot_function.hpp>
 #include <mlpack/methods/ann/activation_functions/elish_function.hpp>
+#include <mlpack/methods/ann/activation_functions/gaussian_function.hpp>
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
@@ -241,6 +242,16 @@ template <
 >
 using GELUFunctionLayer = BaseLayer<
     ActivationFunction, InputDataType, OutputDataType>;
+/**
+ * Standard Gaussian-Layer using the Gaussian activation function.
+ */
+template <
+    class ActivationFunction = GaussianFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using GaussianFunctionLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
 
 /**
  * Standard Elliot-Layer using the Elliot activation function.

src/mlpack/methods/ann/layer/layer.hpp

@@ -55,6 +55,10 @@
 #include "noisylinear.hpp"
 #include "padding.hpp"
 #include "parametric_relu.hpp"
+#include "gru.hpp"
+#include "fast_lstm.hpp"
+#include "radial_basis_function.hpp"
+#include "recurrent.hpp"
 #include "recurrent_attention.hpp"
 #include "recurrent.hpp"
 #include "reinforce_normal.hpp"

src/mlpack/methods/ann/layer/radial_basis_function.hpp

@@ -47,7 +47,8 @@ namespace ann /** Artificial Neural Network. */ {
  */
 template <
     typename InputDataType = arma::mat,
-    typename OutputDataType = arma::mat
+    typename OutputDataType = arma::mat,
+    typename RegularizerType = NoRegularizer
 >
 class RBF
 {
@@ -95,18 +96,6 @@ class RBF
                 ErrorType&& gy,
                 GradientType&& g);
 
-  /*
-   * Calculate the gradient using the output delta and the input activation.
-   *
-   * @param input The input parameter used for calculating the gradient.
-   * @param error The calculated error.
-   * @param gradient The calculated gradient.
-   */
-  template<typename InputType, typename ErrorType, typename GradientType>
-  void Gradient(InputType&& input,
-                ErrorType&& error,
-                GradientType&& gradient);
-
   /*
    * Resets the cell to accept a new input.
    * This breaks the BPTT chain starts a new one.
@@ -159,6 +148,9 @@ class RBF
   //! Locally-stored the learnable scaling factor of the shape.
   InputDataType sigmas;
 
+  //! Locally-stored the outeput distances of the shape.
+  InputDataType distances;
+
   //! Locally-stored number of input units.
   size_t inSize;
 
@@ -212,5 +204,6 @@ class RBF
 } // namespace mlpack
 
 // Include implementation.
+#include "radial_basis_function_impl.hpp"
 
 #endif

src/mlpack/methods/ann/layer/radial_basis_function_impl.hpp

@@ -19,7 +19,8 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 RBF<InputDataType, OutputDataType>::RBF() :
     inSize(0),
     outSize(0)
@@ -27,7 +28,8 @@ RBF<InputDataType, OutputDataType>::RBF() :
   // Nothing to do here.
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 RBF<InputDataType, OutputDataType, RegularizerType>::RBF(
     const size_t inSize,
     const size_t outSize) :
@@ -37,7 +39,8 @@ RBF<InputDataType, OutputDataType, RegularizerType>::RBF(
  // Nothing to do here.
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 void RBF<InputDataType, OutputDataType, RegularizerType>::Reset()
 {
   centres = arma::randu(outSize, inSize);
@@ -45,9 +48,9 @@ void RBF<InputDataType, OutputDataType, RegularizerType>::Reset()
   sigmas = arma::ones(outSize);
 }
 
-template<typename InputDataType, typename OutputDataType>
-template<typename eT>
-void RBF<InputDataType, OutputDataType>::Forward(
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
+void RBF<InputDataType, OutputDataType, RegularizerType>::Forward(
     const InputDataType&& input, OutputDataType&& output)
 {  
   arma::cube x = arma::cube(input.n_rows, outSize, inSize);
@@ -63,28 +66,21 @@ void RBF<InputDataType, OutputDataType>::Forward(
   {
     input.slice(i)= centres;
   }
-  output = arma::pow (arma::sum (arma::pow ((x - c), 2), 1), 0.5) * sigmas;
+  distances = arma::pow (arma::sum (arma::pow ((x - c), 2), 1), 0.5) * sigmas;
+  output = distances;
 
 }
-
-template<typename InputDataType, typename OutputDataType>
-template<typename eT>
-void RBF<InputDataType, OutputDataType>::Backward(
-    const arma::Mat<eT>&& /* input */, arma::Mat<eT>&& gy, arma::Mat<eT>&& g)
-{
-
-}
-
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 template<typename InputType, typename ErrorType, typename GradientType>
-void RBF<InputDataType, OutputDataType>::Gradient(
-    const InputType&& input,
-    ErrorType&& error,
-    GradientType&& gradient)
+void RBF<InputDataType, OutputDataType, RegularizerType>::Backward(
+    const InputDataType&& /* input */, ErrorType&& gy, GradientType&& g)
 {
-
+  g = distances.t() * gy;
 }
 
-template<typename InputDataType, typename OutputDataType>
+template<typename InputDataType, typename OutputDataType,
+    typename RegularizerType>
 template<typename Archive>
 void RBF<InputDataType, OutputDataType, RegularizerType>::serialize(
     Archive& ar, const unsigned int /* version */)