fix indentation errors
This commit is contained in:
dakshitagrawal97
@@ -26,31 +26,31 @@ namespace mlpack {
|
||||
namespace ann /** Artificial Neural Network. */ {
|
||||
|
||||
/**
|
||||
* The alpha - dropout layer is a regularizer that randomly with probability 'ratio'
|
||||
* sets input values to alpha_dash. An affine transformation is applied to the inputs.
|
||||
* In the deterministic mode (during testing), the layer just gives the output.
|
||||
*
|
||||
* Note: During training you should set deterministic to false and during
|
||||
* testing you should set deterministic to true.
|
||||
*
|
||||
* For more information, see the following.
|
||||
*
|
||||
* @code
|
||||
* @article{Klambauer2017,
|
||||
* author = {Gunter Klambauer and Thomas Unterthiner and
|
||||
* Andreas Mayr},
|
||||
* title = {Self-Normalizing Neural Networks},
|
||||
* journal = {Advances in Neural Information Processing Systems},
|
||||
* year = {2017}
|
||||
* }
|
||||
* }
|
||||
* @endcode
|
||||
*
|
||||
* @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
|
||||
* arma::sp_mat or arma::cube).
|
||||
* @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
|
||||
* arma::sp_mat or arma::cube).
|
||||
*/
|
||||
* The alpha - dropout layer is a regularizer that randomly with probability 'ratio'
|
||||
* sets input values to alpha_dash. An affine transformation is applied to the inputs.
|
||||
* In the deterministic mode (during testing), the layer just gives the output.
|
||||
*
|
||||
* Note: During training you should set deterministic to false and during
|
||||
* testing you should set deterministic to true.
|
||||
*
|
||||
* For more information, see the following.
|
||||
*
|
||||
* @code
|
||||
* @article{Klambauer2017,
|
||||
* author = {Gunter Klambauer and Thomas Unterthiner and
|
||||
* Andreas Mayr},
|
||||
* title = {Self-Normalizing Neural Networks},
|
||||
* journal = {Advances in Neural Information Processing Systems},
|
||||
* year = {2017}
|
||||
* }
|
||||
* }
|
||||
* @endcode
|
||||
*
|
||||
* @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
|
||||
* arma::sp_mat or arma::cube).
|
||||
* @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
|
||||
* arma::sp_mat or arma::cube).
|
||||
*/
|
||||
template <
|
||||
typename InputDataType = arma::mat,
|
||||
typename OutputDataType = arma::mat
|
||||
@@ -58,118 +58,119 @@ template <
|
||||
class alphaDropout
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* Create the Alpha_Dropout object using the specified ratio.
|
||||
*
|
||||
* @param ratio The probability of setting a value to alpha_dash.
|
||||
*/
|
||||
alphaDropout(const double ratio = 0.5);
|
||||
/**
|
||||
* Create the Alpha_Dropout object using the specified ratio.
|
||||
*
|
||||
* @param ratio The probability of setting a value to alpha_dash.
|
||||
*/
|
||||
alphaDropout(const double ratio = 0.5);
|
||||
|
||||
/**
|
||||
* Ordinary feed forward pass of the alpha_dropout layer.
|
||||
*
|
||||
* @param input Input data used for evaluating the specified function.
|
||||
* @param output Resulting output activation.
|
||||
*/
|
||||
template<typename eT>
|
||||
void Forward(const arma::Mat<eT>&& input, arma::Mat<eT>&& output);
|
||||
/**
|
||||
* Ordinary feed forward pass of the alpha_dropout layer.
|
||||
*
|
||||
* @param input Input data used for evaluating the specified function.
|
||||
* @param output Resulting output activation.
|
||||
*/
|
||||
template<typename eT>
|
||||
void Forward(const arma::Mat<eT>&& input, arma::Mat<eT>&& output);
|
||||
|
||||
/**
|
||||
* Ordinary feed backward pass of the alpha_dropout layer.
|
||||
*
|
||||
* @param input The propagated input activation.
|
||||
* @param gy The backpropagated error.
|
||||
* @param g The calculated gradient.
|
||||
*/
|
||||
template<typename eT>
|
||||
void Backward(const arma::Mat<eT>&& /* input */,
|
||||
arma::Mat<eT>&& gy,
|
||||
arma::Mat<eT>&& g);
|
||||
/**
|
||||
* Ordinary feed backward pass of the alpha_dropout layer.
|
||||
*
|
||||
* @param input The propagated input activation.
|
||||
* @param gy The backpropagated error.
|
||||
* @param g The calculated gradient.
|
||||
*/
|
||||
template<typename eT>
|
||||
void Backward(
|
||||
const arma::Mat<eT>&& /* input */,
|
||||
arma::Mat<eT>&& gy,
|
||||
arma::Mat<eT>&& g);
|
||||
|
||||
//! Get the input parameter.
|
||||
InputDataType const& InputParameter() const { return inputParameter; }
|
||||
//! Modify the input parameter.
|
||||
InputDataType& InputParameter() { return inputParameter; }
|
||||
//! Get the input parameter.
|
||||
InputDataType const& InputParameter() const { return inputParameter; }
|
||||
//! Modify the input parameter.
|
||||
InputDataType& InputParameter() { return inputParameter; }
|
||||
|
||||
//! Get the output parameter.
|
||||
OutputDataType const& OutputParameter() const { return outputParameter; }
|
||||
//! Modify the output parameter.
|
||||
OutputDataType& OutputParameter() { return outputParameter; }
|
||||
//! Get the output parameter.
|
||||
OutputDataType const& OutputParameter() const { return outputParameter; }
|
||||
//! Modify the output parameter.
|
||||
OutputDataType& OutputParameter() { return outputParameter; }
|
||||
|
||||
//! Get the detla.
|
||||
OutputDataType const& Delta() const { return delta; }
|
||||
//! Modify the delta.
|
||||
OutputDataType& Delta() { return delta; }
|
||||
//! Get the detla.
|
||||
OutputDataType const& Delta() const { return delta; }
|
||||
//! Modify the delta.
|
||||
OutputDataType& Delta() { return delta; }
|
||||
|
||||
//! The value of the deterministic parameter.
|
||||
bool Deterministic() const { return deterministic; }
|
||||
//! Modify the value of the deterministic parameter.
|
||||
bool& Deterministic() { return deterministic; }
|
||||
//! The value of the deterministic parameter.
|
||||
bool Deterministic() const { return deterministic; }
|
||||
//! Modify the value of the deterministic parameter.
|
||||
bool& Deterministic() { return deterministic; }
|
||||
|
||||
//! The probability of setting a value to alpha_dash.
|
||||
double Ratio() const { return ratio; }
|
||||
//! The probability of setting a value to alpha_dash.
|
||||
double Ratio() const { return ratio; }
|
||||
|
||||
//! Value to be multiplied with x for affine transformation.
|
||||
double A() const { return a; }
|
||||
//! Value to be multiplied with x for affine transformation.
|
||||
double A() const { return a; }
|
||||
|
||||
//! Value to be added to a*x for affine transformation.
|
||||
double B() const { return b; }
|
||||
//! Value to be added to a*x for affine transformation.
|
||||
double B() const { return b; }
|
||||
|
||||
//! Value of alpha_dash.
|
||||
double Alpha_Dash() const {return alpha_dash; }
|
||||
//! Value of alpha_dash.
|
||||
double Alpha_Dash() const {return alpha_dash; }
|
||||
|
||||
//! Get the mask.
|
||||
OutputDataType const& Mask() const {return mask;}
|
||||
//! Get the mask.
|
||||
OutputDataType const& Mask() const {return mask;}
|
||||
|
||||
//! Modify the probability of setting a value to alpha_dash. As
|
||||
//! 'a' and 'b' depend on 'ratio', modify them as well.
|
||||
void Ratio(const double r)
|
||||
{
|
||||
ratio = r;
|
||||
a = pow((1 - ratio) * (1 + ratio * pow(alpha_dash, 2)), -0.5);
|
||||
b = -a * alpha_dash * ratio;
|
||||
}
|
||||
//! Modify the probability of setting a value to alpha_dash. As
|
||||
//! 'a' and 'b' depend on 'ratio', modify them as well.
|
||||
void Ratio(const double r)
|
||||
{
|
||||
ratio = r;
|
||||
a = pow((1 - ratio) * (1 + ratio * pow(alpha_dash, 2)), -0.5);
|
||||
b = -a * alpha_dash * ratio;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Serialize the layer.
|
||||
*/
|
||||
template<typename Archive>
|
||||
void serialize(Archive& ar, const unsigned int /* version */);
|
||||
/**
|
||||
* Serialize the layer.
|
||||
*/
|
||||
template<typename Archive>
|
||||
void serialize(Archive& ar, const unsigned int /* version */);
|
||||
|
||||
private:
|
||||
//! Locally-stored delta object.
|
||||
OutputDataType delta;
|
||||
//! Locally-stored delta object.
|
||||
OutputDataType delta;
|
||||
|
||||
//! Locally-stored input parameter object.
|
||||
InputDataType inputParameter;
|
||||
//! Locally-stored input parameter object.
|
||||
InputDataType inputParameter;
|
||||
|
||||
//! Locally-stored output parameter object.
|
||||
OutputDataType outputParameter;
|
||||
//! Locally-stored output parameter object.
|
||||
OutputDataType outputParameter;
|
||||
|
||||
//! Locally-stored mast object.
|
||||
OutputDataType mask;
|
||||
//! Locally-stored mast object.
|
||||
OutputDataType mask;
|
||||
|
||||
//! The probability of setting a value to aplha_dash.
|
||||
double ratio;
|
||||
//! The probability of setting a value to aplha_dash.
|
||||
double ratio;
|
||||
|
||||
//! If true dropout and scaling is disabled, see notes above.
|
||||
bool deterministic;
|
||||
//! If true dropout and scaling is disabled, see notes above.
|
||||
bool deterministic;
|
||||
|
||||
//! Value of alpha for normalized inputs (taken from SELU)
|
||||
const double alpha = 1.6732632423543772848170429916717;
|
||||
//! Value of alpha for normalized inputs (taken from SELU)
|
||||
const double alpha = 1.6732632423543772848170429916717;
|
||||
|
||||
//! Value of lambda for normalized inputs (taken from SELU)
|
||||
const double lambda = 1.0507009873554804934193349852946;
|
||||
//! Value of lambda for normalized inputs (taken from SELU)
|
||||
const double lambda = 1.0507009873554804934193349852946;
|
||||
|
||||
//! The low variance value of SELU activation function.
|
||||
double alpha_dash = -alpha*lambda;
|
||||
//! The low variance value of SELU activation function.
|
||||
double alpha_dash = -alpha*lambda;
|
||||
|
||||
//! Value to be multiplied with x for affine transformation.
|
||||
double a;
|
||||
//! Value to be multiplied with x for affine transformation.
|
||||
double a;
|
||||
|
||||
//! Value to be added to a*x for affine transformation.
|
||||
double b;
|
||||
//! Value to be added to a*x for affine transformation.
|
||||
double b;
|
||||
}; // class Alpha_Dropout
|
||||
|
||||
} // namespace ann
|
||||
|
||||
@@ -24,56 +24,56 @@ namespace ann /** Artificial Neural Network. */ {
|
||||
|
||||
template<typename InputDataType, typename OutputDataType>
|
||||
alphaDropout<InputDataType, OutputDataType>::alphaDropout(
|
||||
const double ratio) :
|
||||
ratio(ratio),
|
||||
deterministic(true)
|
||||
const double ratio) :
|
||||
ratio(ratio),
|
||||
deterministic(true)
|
||||
{
|
||||
Ratio(ratio);
|
||||
Ratio(ratio);
|
||||
}
|
||||
|
||||
template<typename InputDataType, typename OutputDataType>
|
||||
template<typename eT>
|
||||
void alphaDropout<InputDataType, OutputDataType>::Forward(
|
||||
const arma::Mat<eT>&& input,
|
||||
arma::Mat<eT>&& output)
|
||||
const arma::Mat<eT>&& input,
|
||||
arma::Mat<eT>&& output)
|
||||
{
|
||||
// The dropout mask will not be multiplied in the deterministic mode
|
||||
// (during testing).
|
||||
if (deterministic)
|
||||
{
|
||||
output = input;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Set values to alpha_dash with probability ratio. Then apply affine
|
||||
// transformation so as to keep mean and variance of outputs to their
|
||||
// original values.
|
||||
// The dropout mask will not be multiplied in the deterministic mode
|
||||
// (during testing).
|
||||
if (deterministic)
|
||||
{
|
||||
output = input;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Set values to alpha_dash with probability ratio. Then apply affine
|
||||
// transformation so as to keep mean and variance of outputs to their
|
||||
// original values.
|
||||
|
||||
mask = arma::randu< arma::Mat<eT> >(input.n_rows, input.n_cols);
|
||||
mask.transform( [&](double val) { return (val > ratio); } );
|
||||
output = (input % mask + alpha_dash * (1 - mask)) * a + b;
|
||||
}
|
||||
mask = arma::randu< arma::Mat<eT> >(input.n_rows, input.n_cols);
|
||||
mask.transform( [&](double val) { return (val > ratio); } );
|
||||
output = (input % mask + alpha_dash * (1 - mask)) * a + b;
|
||||
}
|
||||
}
|
||||
|
||||
template<typename InputDataType, typename OutputDataType>
|
||||
template<typename eT>
|
||||
void alphaDropout<InputDataType, OutputDataType>::Backward(
|
||||
const arma::Mat<eT>&& /* input */,
|
||||
arma::Mat<eT>&& gy,
|
||||
arma::Mat<eT>&& g)
|
||||
const arma::Mat<eT>&& /* input */,
|
||||
arma::Mat<eT>&& gy,
|
||||
arma::Mat<eT>&& g)
|
||||
{
|
||||
g = gy % mask * a;
|
||||
g = gy % mask * a;
|
||||
}
|
||||
|
||||
template<typename InputDataType, typename OutputDataType>
|
||||
template<typename Archive>
|
||||
void alphaDropout<InputDataType, OutputDataType>::serialize(
|
||||
Archive& ar,
|
||||
const unsigned int /* version */)
|
||||
Archive& ar,
|
||||
const unsigned int /* version */)
|
||||
{
|
||||
ar & BOOST_SERIALIZATION_NVP(ratio);
|
||||
ar & BOOST_SERIALIZATION_NVP(a);
|
||||
ar & BOOST_SERIALIZATION_NVP(b);
|
||||
ar & BOOST_SERIALIZATION_NVP(ratio);
|
||||
ar & BOOST_SERIALIZATION_NVP(a);
|
||||
ar & BOOST_SERIALIZATION_NVP(b);
|
||||
}
|
||||
|
||||
} // namespace ann
|
||||
|
||||
@@ -450,35 +450,34 @@ BOOST_AUTO_TEST_CASE(SimpleAlphaDropoutLayerTest)
|
||||
*/
|
||||
BOOST_AUTO_TEST_CASE(AlphaDropoutProbabilityTest)
|
||||
{
|
||||
arma::mat input = arma::ones(1500, 1);
|
||||
const size_t iterations = 10;
|
||||
arma::mat input = arma::ones(1500, 1);
|
||||
const size_t iterations = 10;
|
||||
|
||||
double probability[5] = { 0.1, 0.3, 0.4, 0.7, 0.8 };
|
||||
for (size_t trial = 0; trial < 5; ++trial)
|
||||
double probability[5] = { 0.1, 0.3, 0.4, 0.7, 0.8 };
|
||||
for (size_t trial = 0; trial < 5; ++trial)
|
||||
{
|
||||
double nonzeroCount = 0;
|
||||
for (size_t i = 0; i < iterations; ++i)
|
||||
{
|
||||
double nonzeroCount = 0;
|
||||
for (size_t i = 0; i < iterations; ++i)
|
||||
{
|
||||
alphaDropout<> module(probability[trial]);
|
||||
module.Deterministic() = false;
|
||||
alphaDropout<> module(probability[trial]);
|
||||
module.Deterministic() = false;
|
||||
|
||||
arma::mat output;
|
||||
module.Forward(std::move(input), std::move(output));
|
||||
arma::mat output;
|
||||
module.Forward(std::move(input), std::move(output));
|
||||
|
||||
// Return a column vector containing the indices of elements of X
|
||||
// that are not alpha_dash, we just need the number of
|
||||
// non_alpha_dash values.
|
||||
arma::uvec non_alpha_dash = arma::find(module.Mask());
|
||||
nonzeroCount += non_alpha_dash.n_elem;
|
||||
}
|
||||
|
||||
const double expected = input.n_elem * (1-probability[trial]) *
|
||||
iterations;
|
||||
|
||||
const double error = fabs(nonzeroCount - expected) / expected;
|
||||
|
||||
BOOST_REQUIRE_LE(error, 0.15);
|
||||
// Return a column vector containing the indices of elements of X
|
||||
// that are not alpha_dash, we just need the number of
|
||||
// non_alpha_dash values.
|
||||
arma::uvec non_alpha_dash = arma::find(module.Mask());
|
||||
nonzeroCount += non_alpha_dash.n_elem;
|
||||
}
|
||||
|
||||
const double expected = input.n_elem * (1-probability[trial]) * iterations;
|
||||
|
||||
const double error = fabs(nonzeroCount - expected) / expected;
|
||||
|
||||
BOOST_REQUIRE_LE(error, 0.15);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -486,14 +485,14 @@ BOOST_AUTO_TEST_CASE(AlphaDropoutProbabilityTest)
|
||||
*/
|
||||
BOOST_AUTO_TEST_CASE(NoAlphaDropoutTest)
|
||||
{
|
||||
arma::mat input = arma::ones(1500, 1);
|
||||
alphaDropout<> module(0);
|
||||
module.Deterministic() = false;
|
||||
arma::mat input = arma::ones(1500, 1);
|
||||
alphaDropout<> module(0);
|
||||
module.Deterministic() = false;
|
||||
|
||||
arma::mat output;
|
||||
module.Forward(std::move(input), std::move(output));
|
||||
arma::mat output;
|
||||
module.Forward(std::move(input), std::move(output));
|
||||
|
||||
BOOST_REQUIRE_EQUAL(arma::accu(output), arma::accu(input));
|
||||
BOOST_REQUIRE_EQUAL(arma::accu(output), arma::accu(input));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user