apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2244 from kartikdutt18/ReFactor-Activation-Functions

Browse Source 
Refactor Activation Function implemented in ann/layers.
This commit is contained in:
Marcus Edel
2020-03-03 12:53:49 +01:00
committed by GitHub
parent 767a82b80b 3660b340a1
commit e9c8c16967
6 changed files with 33 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mlpack/methods/ann/layer/elu.hpp
-63
View File
@@ -173,69 +173,6 @@ class ELU
void serialize(Archive& ar, const unsigned int /* version */);
private:
/**
* Computes the value of activation function.
*
* @param x Input data.
* @return f(x).
*/
double Fn(const double x)
{
if (x < DBL_MAX)
{
return (x > 0) ? lambda * x : lambda * alpha * (std::exp(x) - 1);
}
return 1.0;
}
/**
* Computes the value of activation function using a dense matrix as input.
*
* @param x Input data.
* @param y The resulting output activation.
*/
template<typename eT>
void Fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
{
y.set_size(arma::size(x));
for (size_t i = 0; i < x.n_elem; i++)
{
y(i) = Fn(x(i));
}
}
/**
* Computes the first derivative of the activation function.
*
* @param x Input data.
* @param y Propagated data f(x).
* @return f'(x)
*/
double Deriv(const double x, const double y)
{
return (x > 0) ? lambda : y + lambda * alpha;
}
/**
* Computes the first derivative of the activation function.
*
* @param x Input data.
* @param y Output activations f(x).
* @param z The resulting derivatives.
*/
template<typename InputType, typename OutputType>
void Deriv(const InputType& x, OutputType& y)
{
derivative.set_size(arma::size(x));
for (size_t i = 0; i < x.n_elem; i++)
{
derivative(i) = Deriv(x(i), y(i));
}
}
//! Locally-stored delta object.
OutputDataType delta;
src/mlpack/methods/ann/layer/elu_impl.hpp
+19 -4
View File
@@ -51,12 +51,27 @@ template<typename InputType, typename OutputType>
void ELU<InputDataType, OutputDataType>::Forward(
const InputType&& input, OutputType&& output)
{
Fn(input, output);
if (!deterministic)
output.set_size(arma::size(input));
for (size_t i = 0; i < input.n_elem; i++)
{
Deriv(input, output);
if (input(i) < DBL_MAX)
{
output(i) = (input(i) > 0) ? lambda * input(i) : lambda *
alpha * (std::exp(input(i)) - 1);
}
else
output(i) = 1.0;
}
if (!deterministic)
{
derivative.set_size(arma::size(input));
for (size_t i = 0; i < input.n_elem; i++)
{
derivative(i) = (input(i) > 0) ? lambda : output(i) +
lambda * alpha;
}
}
}
template<typename InputDataType, typename OutputDataType>
src/mlpack/methods/ann/layer/leaky_relu.hpp
-52
View File
@@ -97,58 +97,6 @@ class LeakyReLU
void serialize(Archive& ar, const unsigned int /* version */);
private:
/**
* Computes the LeakyReLU function
*
* @param x Input data.
* @return f(x).
*/
double Fn(const double x)
{
return std::max(x, alpha * x);
}
/**
* Computes the LeakyReLU function using a dense matrix as input.
*
* @param x Input data.
* @param y The resulting output activation.
*/
template<typename eT>
void Fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
{
y = arma::max(x, alpha * x);
}
/**
* Computes the first derivative of the LeakyReLU function.
*
* @param x Input data.
* @return f'(x)
*/
double Deriv(const double x)
{
return (x >= 0) ? 1 : alpha;
}
/**
* Computes the first derivative of the LeakyReLU function.
*
* @param x Input activations.
* @param y The resulting derivatives.
*/
template<typename InputType, typename OutputType>
void Deriv(const InputType& x, OutputType& y)
{
y.set_size(arma::size(x));
for (size_t i = 0; i < x.n_elem; i++)
{
y(i) = Deriv(x(i));
}
}
//! Locally-stored delta object.
OutputDataType delta;
src/mlpack/methods/ann/layer/leaky_relu_impl.hpp
+5 -2
View File
@@ -32,7 +32,7 @@ template<typename InputType, typename OutputType>
void LeakyReLU<InputDataType, OutputDataType>::Forward(
const InputType&& input, OutputType&& output)
{
Fn(input, output);
output = arma::max(input, alpha * input);
}
template<typename InputDataType, typename OutputDataType>
@@ -41,7 +41,10 @@ void LeakyReLU<InputDataType, OutputDataType>::Backward(
const DataType&& input, DataType&& gy, DataType&& g)
{
DataType derivative;
Deriv(input, derivative);
derivative.set_size(arma::size(input));
for (size_t i = 0; i < input.n_elem; i++)
derivative(i) = (input(i) >= 0) ? 1 : alpha;
g = gy % derivative;
}
src/mlpack/methods/ann/layer/parametric_relu.hpp
-54
View File
@@ -126,60 +126,6 @@ class PReLU
void serialize(Archive& ar, const unsigned int /* version */);
private:
/**
* Computes the parametric ReLU function.
*
* @param x Input data.
* @return f(x).
*/
double Fn(const double x)
{
return std::max(x, alpha(0) * x);
}
/**
* Computes the parametric ReLU function using a dense matrix as input.
*
* @param x Input data.
* @param y The resulting output activation.
*/
template<typename eT>
void Fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
{
y = x;
arma::uvec negative = arma::find(x < 0);
y(negative) = x(negative) * alpha(0);
}
/**
* Computes the first derivative of the parametric ReLU function.
*
* @param x Input data.
* @return f'(x)
*/
double Deriv(const double x)
{
return (x >= 0) ? 1 : alpha(0);
}
/**
* Computes the first derivative of the PReLU function.
*
* @param x Input activations.
* @param y The resulting derivatives.
*/
template<typename InputType, typename OutputType>
void Deriv(const InputType& x, OutputType& y)
{
y.set_size(arma::size(x));
for (size_t i = 0; i < x.n_elem; i++)
{
y(i) = Deriv(x(i));
}
}
//! Locally-stored delta object.
OutputDataType delta;
src/mlpack/methods/ann/layer/parametric_relu_impl.hpp
+9 -2
View File
@@ -41,7 +41,9 @@ template<typename InputType, typename OutputType>
void PReLU<InputDataType, OutputDataType>::Forward(
const InputType&& input, OutputType&& output)
{
Fn(input, output);
output = input;
arma::uvec negative = arma::find(input < 0);
output(negative) = input(negative) * alpha(0);
}
template<typename InputDataType, typename OutputDataType>
@@ -50,7 +52,12 @@ void PReLU<InputDataType, OutputDataType>::Backward(
const DataType&& input, DataType&& gy, DataType&& g)
{
DataType derivative;
Deriv(input, derivative);
derivative.set_size(arma::size(input));
for (size_t i = 0; i < input.n_elem; i++)
{
derivative(i) = (input(i) >= 0) ? 1 : alpha(0);
}
g = gy % derivative;
}