From 641fb2c0745ba28a8ca8fac3d87906b802f12095 Mon Sep 17 00:00:00 2001 From: himanshupathak21061998 Date: Thu, 21 May 2020 05:34:24 +0530 Subject: [PATCH] Changing implementation and removing gradient function --- HISTORY.md | 2 + .../gaussian_function.hpp | 6 +- src/mlpack/methods/ann/layer/base_layer.hpp | 2 +- src/mlpack/methods/ann/layer/layer_types.hpp | 7 +- .../ann/layer/radial_basis_function.hpp | 87 +++++++++---------- .../ann/layer/radial_basis_function_impl.hpp | 77 ++++++++-------- .../tests/activation_functions_test.cpp | 28 +++++- src/mlpack/tests/feedforward_network_test.cpp | 38 +++++--- 8 files changed, 146 insertions(+), 101 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 0b2dfe09ea..5da6485129 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -30,6 +30,8 @@ * Fix incorrect neighbors for `k > 1` searches in `approx_kfn` binding, for the `QDAFN` algorithm (#2448). + * Add `RBF` layer in ann module to make `RBFN` architecture (#2261). + ### mlpack 3.3.1 ###### 2020-04-29 * Minor Julia and Python documentation fixes (#2373). diff --git a/src/mlpack/methods/ann/activation_functions/gaussian_function.hpp b/src/mlpack/methods/ann/activation_functions/gaussian_function.hpp index cd311d5aaa..650f80cae9 100644 --- a/src/mlpack/methods/ann/activation_functions/gaussian_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/gaussian_function.hpp @@ -21,8 +21,8 @@ namespace ann /** Artificial Neural Network. */ { * The gaussian function, defined by * * @f{eqnarray*}{ - * f(x) &=& \frac{1}{e^{-1 * x^2}} \\ - * f'(x) &=& f(x) * -x * 2) + * f(x) = e^{-1 * x^2} + * f'(x) = 2 * -x * f(x) * @f} */ class GaussianFunction @@ -55,7 +55,7 @@ class GaussianFunction /** * Computes the first derivative of the gaussian function. * - * @param x Input data. + * @param y Input data. * @return f'(x) */ static double Deriv(const double y) diff --git a/src/mlpack/methods/ann/layer/base_layer.hpp b/src/mlpack/methods/ann/layer/base_layer.hpp index 5309cfffd1..c7aa3887dd 100644 --- a/src/mlpack/methods/ann/layer/base_layer.hpp +++ b/src/mlpack/methods/ann/layer/base_layer.hpp @@ -267,7 +267,7 @@ using ElishFunctionLayer = BaseLayer< ActivationFunction, InputDataType, OutputDataType>; /** - * Standard ELiSH-Layer using the ELiSH activation function. + * Standard Gaussian-Layer using the Gaussian activation function. */ template < class ActivationFunction = GaussianFunction, diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 353da3f138..0f52a24df7 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -82,7 +82,8 @@ template + typename OutputDataType, + typename Activation> class RBF; template*, VRClassReward*, VirtualBatchNorm*, - RBF* + RBF*, + BaseLayer* >; template @@ -234,7 +236,6 @@ using LayerTypes = boost::variant< BaseLayer*, BaseLayer*, BaseLayer*, - BaseLayer*, BatchNorm*, BilinearInterpolation*, CELU*, diff --git a/src/mlpack/methods/ann/layer/radial_basis_function.hpp b/src/mlpack/methods/ann/layer/radial_basis_function.hpp index ba9a27b8ea..6acd159248 100644 --- a/src/mlpack/methods/ann/layer/radial_basis_function.hpp +++ b/src/mlpack/methods/ann/layer/radial_basis_function.hpp @@ -14,7 +14,7 @@ #define MLPACK_METHODS_ANN_LAYER_RBF_HPP #include -#include +#include #include "layer_types.hpp" @@ -22,22 +22,33 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -/** /** * Implementation of the Radial Basis Function layer. The RBF class when use with a * non-linear activation function acts as a Radial Basis Function which can be used * with Feed-Forward neural network. * + * For more information, refer to the following paper, + * + * @code + * @article{Volume 51: Artificial Intelligence and Statistics, + * author = {Qichao Que, Mikhail Belkin}, + * title = {Back to the Future: Radial Basis Function Networks Revisited}, + * year = {2016}, + * url = {http://proceedings.mlr.press/v51/que16.pdf}, + * } + * @endcode * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). + * @tparam Activation Type of the activation function (mlpack::ann::Gaussian). */ template < typename InputDataType = arma::mat, - typename OutputDataType = arma::mat + typename OutputDataType = arma::mat, + typename Activation = GaussianFunction > class RBF { @@ -51,15 +62,13 @@ class RBF * * @param inSize The number of input units. * @param outSize The number of output units. + * @param centres The centres calculated using k-means of data. + * @param betas The beta value to be used with centres. */ RBF(const size_t inSize, const size_t outSize, - arma::mat& centres); - - /** - * Reset the layer parameter. - */ - void Reset(); + arma::mat& centres, + double betas = 0); /** * Ordinary feed forward pass of the radial basis function. @@ -73,26 +82,11 @@ class RBF /** * Ordinary feed backward pass of the radial basis function. * - * @param input The propagated input activation. - * @param gy The backpropagated error. - * @param g The calculated gradient. */ template void Backward(const arma::Mat& /* input */, - const arma::Mat& gy, - arma::Mat& g); - - /* - * Calculate the gradient using the output delta and the input activation. - * - * @param input The input parameter used for calculating the gradient. - * @param error The calculated error. - * @param gradient The calculated gradient. - */ - template - void Gradient(const arma::Mat& input, - const arma::Mat& error, - arma::Mat& gradient); + const arma::Mat& /* gy */, + arma::Mat& /* g */); //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } @@ -123,33 +117,32 @@ class RBF void serialize(Archive& ar, const unsigned int /* version */); private: - //! Locally-stored delta object. - OutputDataType delta; - - //! Locally-stored output parameter object. - OutputDataType outputParameter; - - //! Locally-stored the learnable centre of the shape. - InputDataType centres; - - //! Locally-stored the learnable scaling factor of the shape. - InputDataType sigmas; - - //! Locally-stored input parameter object. - InputDataType inputParameter; - - //! Locally-stored the output distances of the shape. - OutputDataType distances; - - //! Locally-stored reset parameter used to initialize the layer once. - bool reset; - //! Locally-stored number of input units. size_t inSize; //! Locally-stored number of output units. size_t outSize; + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Locally-stored the sigmas values. + double sigmas; + + //! Locally-stored the betas values. + double betas; + + //! Locally-stored the learnable centre of the shape. + InputDataType centres; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored the output distances of the shape. + OutputDataType distances; }; // class RBF } // namespace ann diff --git a/src/mlpack/methods/ann/layer/radial_basis_function_impl.hpp b/src/mlpack/methods/ann/layer/radial_basis_function_impl.hpp index 90d2c4e73a..8ac455bed2 100644 --- a/src/mlpack/methods/ann/layer/radial_basis_function_impl.hpp +++ b/src/mlpack/methods/ann/layer/radial_basis_function_impl.hpp @@ -1,5 +1,5 @@ /** - * @file radial_basis_impl.hpp + * @file radial_basis_function_impl.hpp * @author Himanshu Pathak * * @@ -12,50 +12,58 @@ #define MLPACK_METHODS_ANN_LAYER_RBF_IMPL_HPP // In case it hasn't yet been included. -#include "dropout.hpp" +#include "radial_basis_function.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -RBF::RBF() : +template +RBF::RBF() : inSize(0), outSize(0), - reset(false) + sigmas(0), + betas(0) { // Nothing to do here. } -template -RBF::RBF( +template +RBF::RBF( const size_t inSize, const size_t outSize, - arma::mat& centres) : + arma::mat& centres, + double betas) : inSize(inSize), outSize(outSize), - centres(centres), - reset(false) + betas(betas), + centres(centres) { + sigmas = 0; + if (betas == 0) + { + for (size_t i = 0; i < centres.n_cols; i++) + { + double max_dis = 0; + arma::mat temp = centres.each_col() - centres.col(i); + max_dis = arma::accu(arma::max(arma::pow(arma::sum( + arma::pow((temp), + 2), 0), 0.5).t())); + if (max_dis > sigmas) + sigmas = max_dis; + } + this->betas = std::pow(2 * outSize, 0.5) / sigmas ; + } } -template -void RBF::Reset() -{ -} - -template +template template -void RBF::Forward( +void RBF::Forward( const arma::Mat& input, arma::Mat& output) { - if(!reset) - { - arma::mat sigmas = arma::mat(1, outSize); - sigmas.ones(); - sigmas = sigmas / outSize; - reset = true; - } distances = arma::mat(outSize, input.n_cols); for (size_t i = 0; i < input.n_cols; i++) @@ -65,28 +73,29 @@ void RBF::Forward( arma::pow((temp), 2), 0), 0.5).t(); } - - output = distances; + Activation::Fn(distances * std::pow(betas, 0.5), + output); } -template + +template template -void RBF::Backward( +void RBF::Backward( const arma::Mat& /* input */, - const arma::Mat& gy, - arma::Mat& g) + const arma::Mat& /* gy */, + arma::Mat& /* g */) { - g = centres.t() * gy; } -template +template template -void RBF::serialize( +void RBF::serialize( Archive& ar, const unsigned int /* version */) { ar & BOOST_SERIALIZATION_NVP(distances); - ar & BOOST_SERIALIZATION_NVP(sigmas); ar & BOOST_SERIALIZATION_NVP(centres); } diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index 041756f34b..406dda4d7b 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include "test_tools.hpp" @@ -975,7 +976,7 @@ BOOST_AUTO_TEST_CASE(QuadraticFunctionTest) CheckActivationCorrect(activationData, desiredActivations); CheckDerivativeCorrect(desiredActivations, - desiredDerivatives); + desiredDerivatives); } /** @@ -1040,4 +1041,29 @@ BOOST_AUTO_TEST_CASE(Poisson1FunctionTest) desiredDerivatives); } +/** + * Basic test of the Gaussian activation function. + */ +BOOST_AUTO_TEST_CASE(GaussianFunctionTest) +{ + const arma::colvec desiredActivations("0.018315639 0.000035713 \ + 1.6052280551856116e-09 \ + 0 0.367879441 0.367879441 \ + 0.018315639 1"); + + const arma::colvec desiredDerivatives("-0.036618991635992616 \ + -0.0000714259999 \ + -0.0000000032104561 \ + 0 -0.6426287436 \ + -0.642628743680 \ + -0.03661899163 \ + -0.73575888234"); + + CheckActivationCorrect(activationData, + desiredActivations); + CheckDerivativeCorrect(desiredActivations, + desiredDerivatives); +} + + BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/feedforward_network_test.cpp b/src/mlpack/tests/feedforward_network_test.cpp index 8a3229a7e2..57e22a16b6 100644 --- a/src/mlpack/tests/feedforward_network_test.cpp +++ b/src/mlpack/tests/feedforward_network_test.cpp @@ -628,6 +628,7 @@ BOOST_AUTO_TEST_CASE(OptimizerTest) */ BOOST_AUTO_TEST_CASE(RBFNetworkTest) { + mlpack::math::RandomSeed(std::time(NULL)); // Load the dataset. arma::mat trainData; data::Load("thyroid_train.csv", trainData, true); @@ -635,6 +636,12 @@ BOOST_AUTO_TEST_CASE(RBFNetworkTest) arma::mat trainLabels = trainData.row(trainData.n_rows - 1); trainData.shed_row(trainData.n_rows - 1); + arma::mat trainLabels1 = arma::zeros(3, trainData.n_cols); + for(size_t i = 0; i < trainData.n_cols; i++) + { + trainLabels1.col(i).row((trainLabels(i) - 1)) = 1; + } + arma::mat testData; data::Load("thyroid_test.csv", testData, true); @@ -659,12 +666,13 @@ BOOST_AUTO_TEST_CASE(RBFNetworkTest) KMeans<> kmeans; kmeans.Cluster(trainData, 8, centroids); - FFN > model; - model.Add >(trainData.n_rows, 8, centroids); + FFN > model; + model.Add >(trainData.n_rows, 8, centroids); model.Add >(8, 3); - model.Add >(); - TestNetwork<>(model, trainData, trainLabels, testData, testLabels, 10, 0.1); + // RBFN neural net with MeanSquaredError. + TestNetwork<>(model, trainData, trainLabels1, testData, testLabels, 10, 0.1); + arma::mat dataset; dataset.load("mnist_first250_training_4s_and_9s.arm"); @@ -676,20 +684,26 @@ BOOST_AUTO_TEST_CASE(RBFNetworkTest) arma::mat labels = arma::zeros(1, dataset.n_cols); labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); + + arma::mat labels1 = arma::zeros(2, dataset.n_cols); + for(size_t i = 0; i < dataset.n_cols; i++) + { + labels1.col(i).row(labels(i)) = 1; + } labels += 1; - + arma::mat centroids1; arma::Row assignments; KMeans<> kmeans1; - kmeans1.Cluster(dataset, 10, centroids1); + kmeans1.Cluster(dataset, 140, centroids1); - FFN > model1; - model1.Add >(dataset.n_rows, 10, centroids1); - model.Add >(10, 2); - model1.Add >(); - // Vanilla neural net with logistic activation function. - TestNetwork<>(model1, dataset, labels, dataset, labels, 10, 0.3); + FFN > model1; + model1.Add >(dataset.n_rows, 140, centroids1, 4.1); + model1.Add >(140, 2); + + // RBFN neural net with MeanSquaredError. + TestNetwork<>(model1, dataset, labels1, dataset, labels, 10, 0.1); } BOOST_AUTO_TEST_SUITE_END();