apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Changing implementation and removing gradient function

Browse Source
This commit is contained in:
himanshupathak21061998
2020-06-13 02:37:09 +05:30
parent 48f68b9d19
commit 641fb2c074
8 changed files with 146 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
HISTORY.md
+2
View File
@@ -30,6 +30,8 @@
* Fix incorrect neighbors for `k > 1` searches in `approx_kfn` binding, for
the `QDAFN` algorithm (#2448).
* Add `RBF` layer in ann module to make `RBFN` architecture (#2261).
### mlpack 3.3.1
###### 2020-04-29
* Minor Julia and Python documentation fixes (#2373).
src/mlpack/methods/ann/activation_functions/gaussian_function.hpp
+3 -3
View File
@@ -21,8 +21,8 @@ namespace ann /** Artificial Neural Network. */ {
* The gaussian function, defined by
*
* @f{eqnarray*}{
* f(x) &=& \frac{1}{e^{-1 * x^2}} \\
* f'(x) &=& f(x) * -x * 2)
* f(x) = e^{-1 * x^2}
* f'(x) = 2 * -x * f(x)
* @f}
*/
class GaussianFunction
@@ -55,7 +55,7 @@ class GaussianFunction
/**
* Computes the first derivative of the gaussian function.
*
* @param x Input data.
* @param y Input data.
* @return f'(x)
*/
static double Deriv(const double y)
src/mlpack/methods/ann/layer/base_layer.hpp
+1 -1
View File
@@ -267,7 +267,7 @@ using ElishFunctionLayer = BaseLayer<
ActivationFunction, InputDataType, OutputDataType>;
/**
* Standard ELiSH-Layer using the ELiSH activation function.
* Standard Gaussian-Layer using the Gaussian activation function.
*/
template <
class ActivationFunction = GaussianFunction,
src/mlpack/methods/ann/layer/layer_types.hpp
+4 -3
View File
@@ -82,7 +82,8 @@ template<typename InputDataType,
class Linear;
template<typename InputDataType,
typename OutputDataType>
typename OutputDataType,
typename Activation>
class RBF;
template<typename InputDataType,
@@ -215,7 +216,8 @@ using MoreTypes = boost::variant<
Subview<arma::mat, arma::mat>*,
VRClassReward<arma::mat, arma::mat>*,
VirtualBatchNorm<arma::mat, arma::mat>*,
RBF<arma::mat, arma::mat>*
RBF<arma::mat, arma::mat, GaussianFunction>*,
BaseLayer<GaussianFunction, arma::mat, arma::mat>*
>;
template <typename... CustomLayers>
@@ -234,7 +236,6 @@ using LayerTypes = boost::variant<
BaseLayer<TanhFunction, arma::mat, arma::mat>*,
BaseLayer<SoftplusFunction, arma::mat, arma::mat>*,
BaseLayer<RectifierFunction, arma::mat, arma::mat>*,
BaseLayer<GaussianFunction, arma::mat, arma::mat>*,
BatchNorm<arma::mat, arma::mat>*,
BilinearInterpolation<arma::mat, arma::mat>*,
CELU<arma::mat, arma::mat>*,
src/mlpack/methods/ann/layer/radial_basis_function.hpp
+40 -47
View File
@@ -14,7 +14,7 @@
#define MLPACK_METHODS_ANN_LAYER_RBF_HPP
#include <mlpack/prereqs.hpp>
#include <mlpack/methods/ann/regularizer/no_regularizer.hpp>
#include <mlpack/methods/ann/activation_functions/gaussian_function.hpp>
#include "layer_types.hpp"
@@ -22,22 +22,33 @@ namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
/**
/**
* Implementation of the Radial Basis Function layer. The RBF class when use with a
* non-linear activation function acts as a Radial Basis Function which can be used
* with Feed-Forward neural network.
*
* For more information, refer to the following paper,
*
* @code
* @article{Volume 51: Artificial Intelligence and Statistics,
* author = {Qichao Que, Mikhail Belkin},
* title = {Back to the Future: Radial Basis Function Networks Revisited},
* year = {2016},
* url = {http://proceedings.mlr.press/v51/que16.pdf},
* }
* @endcode
*
* @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
* arma::sp_mat or arma::cube).
* @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
* arma::sp_mat or arma::cube).
* @tparam Activation Type of the activation function (mlpack::ann::Gaussian).
*/
template <
typename InputDataType = arma::mat,
typename OutputDataType = arma::mat
typename OutputDataType = arma::mat,
typename Activation = GaussianFunction
>
class RBF
{
@@ -51,15 +62,13 @@ class RBF
*
* @param inSize The number of input units.
* @param outSize The number of output units.
* @param centres The centres calculated using k-means of data.
* @param betas The beta value to be used with centres.
*/
RBF(const size_t inSize,
const size_t outSize,
arma::mat& centres);
/**
* Reset the layer parameter.
*/
void Reset();
arma::mat& centres,
double betas = 0);
/**
* Ordinary feed forward pass of the radial basis function.
@@ -73,26 +82,11 @@ class RBF
/**
* Ordinary feed backward pass of the radial basis function.
*
* @param input The propagated input activation.
* @param gy The backpropagated error.
* @param g The calculated gradient.
*/
template<typename eT>
void Backward(const arma::Mat<eT>& /* input */,
const arma::Mat<eT>& gy,
arma::Mat<eT>& g);
/*
* Calculate the gradient using the output delta and the input activation.
*
* @param input The input parameter used for calculating the gradient.
* @param error The calculated error.
* @param gradient The calculated gradient.
*/
template<typename eT>
void Gradient(const arma::Mat<eT>& input,
const arma::Mat<eT>& error,
arma::Mat<eT>& gradient);
const arma::Mat<eT>& /* gy */,
arma::Mat<eT>& /* g */);
//! Get the output parameter.
OutputDataType const& OutputParameter() const { return outputParameter; }
@@ -123,33 +117,32 @@ class RBF
void serialize(Archive& ar, const unsigned int /* version */);
private:
//! Locally-stored delta object.
OutputDataType delta;
//! Locally-stored output parameter object.
OutputDataType outputParameter;
//! Locally-stored the learnable centre of the shape.
InputDataType centres;
//! Locally-stored the learnable scaling factor of the shape.
InputDataType sigmas;
//! Locally-stored input parameter object.
InputDataType inputParameter;
//! Locally-stored the output distances of the shape.
OutputDataType distances;
//! Locally-stored reset parameter used to initialize the layer once.
bool reset;
//! Locally-stored number of input units.
size_t inSize;
//! Locally-stored number of output units.
size_t outSize;
//! Locally-stored delta object.
OutputDataType delta;
//! Locally-stored output parameter object.
OutputDataType outputParameter;
//! Locally-stored the sigmas values.
double sigmas;
//! Locally-stored the betas values.
double betas;
//! Locally-stored the learnable centre of the shape.
InputDataType centres;
//! Locally-stored input parameter object.
InputDataType inputParameter;
//! Locally-stored the output distances of the shape.
OutputDataType distances;
}; // class RBF
} // namespace ann
src/mlpack/methods/ann/layer/radial_basis_function_impl.hpp
+43 -34
View File
@@ -1,5 +1,5 @@
/**
* @file radial_basis_impl.hpp
* @file radial_basis_function_impl.hpp
* @author Himanshu Pathak
*
*
@@ -12,50 +12,58 @@
#define MLPACK_METHODS_ANN_LAYER_RBF_IMPL_HPP
// In case it hasn't yet been included.
#include "dropout.hpp"
#include "radial_basis_function.hpp"
namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
template<typename InputDataType, typename OutputDataType>
RBF<InputDataType, OutputDataType>::RBF() :
template<typename InputDataType, typename OutputDataType,
typename Activation>
RBF<InputDataType, OutputDataType, Activation>::RBF() :
inSize(0),
outSize(0),
reset(false)
sigmas(0),
betas(0)
{
// Nothing to do here.
}
template<typename InputDataType, typename OutputDataType>
RBF<InputDataType, OutputDataType>::RBF(
template<typename InputDataType, typename OutputDataType,
typename Activation>
RBF<InputDataType, OutputDataType, Activation>::RBF(
const size_t inSize,
const size_t outSize,
arma::mat& centres) :
arma::mat& centres,
double betas) :
inSize(inSize),
outSize(outSize),
centres(centres),
reset(false)
betas(betas),
centres(centres)
{
sigmas = 0;
if (betas == 0)
{
for (size_t i = 0; i < centres.n_cols; i++)
{
double max_dis = 0;
arma::mat temp = centres.each_col() - centres.col(i);
max_dis = arma::accu(arma::max(arma::pow(arma::sum(
arma::pow((temp),
2), 0), 0.5).t()));
if (max_dis > sigmas)
sigmas = max_dis;
}
this->betas = std::pow(2 * outSize, 0.5) / sigmas ;
}
}
template<typename InputDataType, typename OutputDataType>
void RBF<InputDataType, OutputDataType>::Reset()
{
}
template<typename InputDataType, typename OutputDataType>
template<typename InputDataType, typename OutputDataType,
typename Activation>
template<typename eT>
void RBF<InputDataType, OutputDataType>::Forward(
void RBF<InputDataType, OutputDataType, Activation>::Forward(
const arma::Mat<eT>& input,
arma::Mat<eT>& output)
{
if(!reset)
{
arma::mat sigmas = arma::mat(1, outSize);
sigmas.ones();
sigmas = sigmas / outSize;
reset = true;
}
distances = arma::mat(outSize, input.n_cols);
for (size_t i = 0; i < input.n_cols; i++)
@@ -65,28 +73,29 @@ void RBF<InputDataType, OutputDataType>::Forward(
arma::pow((temp),
2), 0), 0.5).t();
}
output = distances;
Activation::Fn(distances * std::pow(betas, 0.5),
output);
}
template<typename InputDataType, typename OutputDataType>
template<typename InputDataType, typename OutputDataType,
typename Activation>
template<typename eT>
void RBF<InputDataType, OutputDataType>::Backward(
void RBF<InputDataType, OutputDataType, Activation>::Backward(
const arma::Mat<eT>& /* input */,
const arma::Mat<eT>& gy,
arma::Mat<eT>& g)
const arma::Mat<eT>& /* gy */,
arma::Mat<eT>& /* g */)
{
g = centres.t() * gy;
}
template<typename InputDataType, typename OutputDataType>
template<typename InputDataType, typename OutputDataType,
typename Activation>
template<typename Archive>
void RBF<InputDataType, OutputDataType>::serialize(
void RBF<InputDataType, OutputDataType, Activation>::serialize(
Archive& ar,
const unsigned int /* version */)
{
ar & BOOST_SERIALIZATION_NVP(distances);
ar & BOOST_SERIALIZATION_NVP(sigmas);
ar & BOOST_SERIALIZATION_NVP(centres);
}
src/mlpack/tests/activation_functions_test.cpp
+27 -1
View File
@@ -31,6 +31,7 @@
#include <mlpack/methods/ann/activation_functions/multi_quadratic_function.hpp>
#include <mlpack/methods/ann/activation_functions/spline_function.hpp>
#include <mlpack/methods/ann/activation_functions/poisson1_function.hpp>
#include <mlpack/methods/ann/activation_functions/gaussian_function.hpp>
#include <boost/test/unit_test.hpp>
#include "test_tools.hpp"
@@ -975,7 +976,7 @@ BOOST_AUTO_TEST_CASE(QuadraticFunctionTest)
CheckActivationCorrect<QuadraticFunction>(activationData, desiredActivations);
CheckDerivativeCorrect<QuadraticFunction>(desiredActivations,
desiredDerivatives);
desiredDerivatives);
}
/**
@@ -1040,4 +1041,29 @@ BOOST_AUTO_TEST_CASE(Poisson1FunctionTest)
desiredDerivatives);
}
/**
* Basic test of the Gaussian activation function.
*/
BOOST_AUTO_TEST_CASE(GaussianFunctionTest)
{
const arma::colvec desiredActivations("0.018315639 0.000035713 \
1.6052280551856116e-09 \
0 0.367879441 0.367879441 \
0.018315639 1");
const arma::colvec desiredDerivatives("-0.036618991635992616 \
-0.0000714259999 \
-0.0000000032104561 \
0 -0.6426287436 \
-0.642628743680 \
-0.03661899163 \
-0.73575888234");
CheckActivationCorrect<GaussianFunction>(activationData,
desiredActivations);
CheckDerivativeCorrect<GaussianFunction>(desiredActivations,
desiredDerivatives);
}
BOOST_AUTO_TEST_SUITE_END();
src/mlpack/tests/feedforward_network_test.cpp
+26 -12
View File
@@ -628,6 +628,7 @@ BOOST_AUTO_TEST_CASE(OptimizerTest)
*/
BOOST_AUTO_TEST_CASE(RBFNetworkTest)
{
mlpack::math::RandomSeed(std::time(NULL));
// Load the dataset.
arma::mat trainData;
data::Load("thyroid_train.csv", trainData, true);
@@ -635,6 +636,12 @@ BOOST_AUTO_TEST_CASE(RBFNetworkTest)
arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
trainData.shed_row(trainData.n_rows - 1);
arma::mat trainLabels1 = arma::zeros(3, trainData.n_cols);
for(size_t i = 0; i < trainData.n_cols; i++)
{
trainLabels1.col(i).row((trainLabels(i) - 1)) = 1;
}
arma::mat testData;
data::Load("thyroid_test.csv", testData, true);
@@ -659,12 +666,13 @@ BOOST_AUTO_TEST_CASE(RBFNetworkTest)
KMeans<> kmeans;
kmeans.Cluster(trainData, 8, centroids);
FFN<NegativeLogLikelihood<> > model;
model.Add<RBF<> >(trainData.n_rows, 8, centroids);
FFN<MeanSquaredError<> > model;
model.Add<RBF<> >(trainData.n_rows, 8, centroids);
model.Add<Linear<> >(8, 3);
model.Add<LogSoftMax<> >();
TestNetwork<>(model, trainData, trainLabels, testData, testLabels, 10, 0.1);
// RBFN neural net with MeanSquaredError.
TestNetwork<>(model, trainData, trainLabels1, testData, testLabels, 10, 0.1);
arma::mat dataset;
dataset.load("mnist_first250_training_4s_and_9s.arm");
@@ -676,20 +684,26 @@ BOOST_AUTO_TEST_CASE(RBFNetworkTest)
arma::mat labels = arma::zeros(1, dataset.n_cols);
labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1);
arma::mat labels1 = arma::zeros(2, dataset.n_cols);
for(size_t i = 0; i < dataset.n_cols; i++)
{
labels1.col(i).row(labels(i)) = 1;
}
labels += 1;
arma::mat centroids1;
arma::Row<size_t> assignments;
KMeans<> kmeans1;
kmeans1.Cluster(dataset, 10, centroids1);
kmeans1.Cluster(dataset, 140, centroids1);
FFN<NegativeLogLikelihood<> > model1;
model1.Add<RBF<> >(dataset.n_rows, 10, centroids1);
model.Add<Linear<> >(10, 2);
model1.Add<LogSoftMax<> >();
// Vanilla neural net with logistic activation function.
TestNetwork<>(model1, dataset, labels, dataset, labels, 10, 0.3);
FFN<MeanSquaredError<> > model1;
model1.Add<RBF<> >(dataset.n_rows, 140, centroids1, 4.1);
model1.Add<Linear<> >(140, 2);
// RBFN neural net with MeanSquaredError.
TestNetwork<>(model1, dataset, labels1, dataset, labels, 10, 0.1);
}
BOOST_AUTO_TEST_SUITE_END();