diff --git a/src/mlpack/methods/ann/layer/alpha_dropout.hpp b/src/mlpack/methods/ann/layer/alpha_dropout.hpp index 20f9802e74..b6d97bd6b1 100644 --- a/src/mlpack/methods/ann/layer/alpha_dropout.hpp +++ b/src/mlpack/methods/ann/layer/alpha_dropout.hpp @@ -26,31 +26,31 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** -* The alpha - dropout layer is a regularizer that randomly with probability 'ratio' -* sets input values to alpha_dash. An affine transformation is applied to the inputs. -* In the deterministic mode (during testing), the layer just gives the output. -* -* Note: During training you should set deterministic to false and during -* testing you should set deterministic to true. -* -* For more information, see the following. -* -* @code -* @article{Klambauer2017, -* author = {Gunter Klambauer and Thomas Unterthiner and -* Andreas Mayr}, -* title = {Self-Normalizing Neural Networks}, -* journal = {Advances in Neural Information Processing Systems}, -* year = {2017} -* } -* } -* @endcode -* -* @tparam InputDataType Type of the input data (arma::colvec, arma::mat, -* arma::sp_mat or arma::cube). -* @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, -* arma::sp_mat or arma::cube). -*/ + * The alpha - dropout layer is a regularizer that randomly with probability 'ratio' + * sets input values to alpha_dash. An affine transformation is applied to the inputs. + * In the deterministic mode (during testing), the layer just gives the output. + * + * Note: During training you should set deterministic to false and during + * testing you should set deterministic to true. + * + * For more information, see the following. + * + * @code + * @article{Klambauer2017, + * author = {Gunter Klambauer and Thomas Unterthiner and + * Andreas Mayr}, + * title = {Self-Normalizing Neural Networks}, + * journal = {Advances in Neural Information Processing Systems}, + * year = {2017} + * } + * } + * @endcode + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ template < typename InputDataType = arma::mat, typename OutputDataType = arma::mat @@ -58,118 +58,119 @@ template < class alphaDropout { public: - /** - * Create the Alpha_Dropout object using the specified ratio. - * - * @param ratio The probability of setting a value to alpha_dash. - */ - alphaDropout(const double ratio = 0.5); + /** + * Create the Alpha_Dropout object using the specified ratio. + * + * @param ratio The probability of setting a value to alpha_dash. + */ + alphaDropout(const double ratio = 0.5); - /** - * Ordinary feed forward pass of the alpha_dropout layer. - * - * @param input Input data used for evaluating the specified function. - * @param output Resulting output activation. - */ - template - void Forward(const arma::Mat&& input, arma::Mat&& output); + /** + * Ordinary feed forward pass of the alpha_dropout layer. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + template + void Forward(const arma::Mat&& input, arma::Mat&& output); - /** - * Ordinary feed backward pass of the alpha_dropout layer. - * - * @param input The propagated input activation. - * @param gy The backpropagated error. - * @param g The calculated gradient. - */ - template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + /** + * Ordinary feed backward pass of the alpha_dropout layer. + * + * @param input The propagated input activation. + * @param gy The backpropagated error. + * @param g The calculated gradient. + */ + template + void Backward( + const arma::Mat&& /* input */, + arma::Mat&& gy, + arma::Mat&& g); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } + //! Get the input parameter. + InputDataType const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. - OutputDataType const& OutputParameter() const { return outputParameter; } - //! Modify the output parameter. - OutputDataType& OutputParameter() { return outputParameter; } + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } - //! Get the detla. - OutputDataType const& Delta() const { return delta; } - //! Modify the delta. - OutputDataType& Delta() { return delta; } + //! Get the detla. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } - //! The value of the deterministic parameter. - bool Deterministic() const { return deterministic; } - //! Modify the value of the deterministic parameter. - bool& Deterministic() { return deterministic; } + //! The value of the deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of the deterministic parameter. + bool& Deterministic() { return deterministic; } - //! The probability of setting a value to alpha_dash. - double Ratio() const { return ratio; } + //! The probability of setting a value to alpha_dash. + double Ratio() const { return ratio; } - //! Value to be multiplied with x for affine transformation. - double A() const { return a; } + //! Value to be multiplied with x for affine transformation. + double A() const { return a; } - //! Value to be added to a*x for affine transformation. - double B() const { return b; } + //! Value to be added to a*x for affine transformation. + double B() const { return b; } - //! Value of alpha_dash. - double Alpha_Dash() const {return alpha_dash; } + //! Value of alpha_dash. + double Alpha_Dash() const {return alpha_dash; } - //! Get the mask. - OutputDataType const& Mask() const {return mask;} + //! Get the mask. + OutputDataType const& Mask() const {return mask;} - //! Modify the probability of setting a value to alpha_dash. As - //! 'a' and 'b' depend on 'ratio', modify them as well. - void Ratio(const double r) - { - ratio = r; - a = pow((1 - ratio) * (1 + ratio * pow(alpha_dash, 2)), -0.5); - b = -a * alpha_dash * ratio; - } + //! Modify the probability of setting a value to alpha_dash. As + //! 'a' and 'b' depend on 'ratio', modify them as well. + void Ratio(const double r) + { + ratio = r; + a = pow((1 - ratio) * (1 + ratio * pow(alpha_dash, 2)), -0.5); + b = -a * alpha_dash * ratio; + } - /** - * Serialize the layer. - */ - template - void serialize(Archive& ar, const unsigned int /* version */); + /** + * Serialize the layer. + */ + template + void serialize(Archive& ar, const unsigned int /* version */); private: - //! Locally-stored delta object. - OutputDataType delta; + //! Locally-stored delta object. + OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; + //! Locally-stored input parameter object. + InputDataType inputParameter; - //! Locally-stored output parameter object. - OutputDataType outputParameter; + //! Locally-stored output parameter object. + OutputDataType outputParameter; - //! Locally-stored mast object. - OutputDataType mask; + //! Locally-stored mast object. + OutputDataType mask; - //! The probability of setting a value to aplha_dash. - double ratio; + //! The probability of setting a value to aplha_dash. + double ratio; - //! If true dropout and scaling is disabled, see notes above. - bool deterministic; + //! If true dropout and scaling is disabled, see notes above. + bool deterministic; - //! Value of alpha for normalized inputs (taken from SELU) - const double alpha = 1.6732632423543772848170429916717; + //! Value of alpha for normalized inputs (taken from SELU) + const double alpha = 1.6732632423543772848170429916717; - //! Value of lambda for normalized inputs (taken from SELU) - const double lambda = 1.0507009873554804934193349852946; + //! Value of lambda for normalized inputs (taken from SELU) + const double lambda = 1.0507009873554804934193349852946; - //! The low variance value of SELU activation function. - double alpha_dash = -alpha*lambda; + //! The low variance value of SELU activation function. + double alpha_dash = -alpha*lambda; - //! Value to be multiplied with x for affine transformation. - double a; + //! Value to be multiplied with x for affine transformation. + double a; - //! Value to be added to a*x for affine transformation. - double b; + //! Value to be added to a*x for affine transformation. + double b; }; // class Alpha_Dropout } // namespace ann diff --git a/src/mlpack/methods/ann/layer/alpha_dropout_impl.hpp b/src/mlpack/methods/ann/layer/alpha_dropout_impl.hpp index 2c7896d4a3..e435751b4b 100644 --- a/src/mlpack/methods/ann/layer/alpha_dropout_impl.hpp +++ b/src/mlpack/methods/ann/layer/alpha_dropout_impl.hpp @@ -24,56 +24,56 @@ namespace ann /** Artificial Neural Network. */ { template alphaDropout::alphaDropout( - const double ratio) : - ratio(ratio), - deterministic(true) + const double ratio) : + ratio(ratio), + deterministic(true) { - Ratio(ratio); + Ratio(ratio); } template template void alphaDropout::Forward( - const arma::Mat&& input, - arma::Mat&& output) + const arma::Mat&& input, + arma::Mat&& output) { - // The dropout mask will not be multiplied in the deterministic mode - // (during testing). - if (deterministic) - { - output = input; - } - else - { - // Set values to alpha_dash with probability ratio. Then apply affine - // transformation so as to keep mean and variance of outputs to their - // original values. + // The dropout mask will not be multiplied in the deterministic mode + // (during testing). + if (deterministic) + { + output = input; + } + else + { + // Set values to alpha_dash with probability ratio. Then apply affine + // transformation so as to keep mean and variance of outputs to their + // original values. - mask = arma::randu< arma::Mat >(input.n_rows, input.n_cols); - mask.transform( [&](double val) { return (val > ratio); } ); - output = (input % mask + alpha_dash * (1 - mask)) * a + b; - } + mask = arma::randu< arma::Mat >(input.n_rows, input.n_cols); + mask.transform( [&](double val) { return (val > ratio); } ); + output = (input % mask + alpha_dash * (1 - mask)) * a + b; + } } template template void alphaDropout::Backward( - const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g) + const arma::Mat&& /* input */, + arma::Mat&& gy, + arma::Mat&& g) { - g = gy % mask * a; + g = gy % mask * a; } template template void alphaDropout::serialize( - Archive& ar, - const unsigned int /* version */) + Archive& ar, + const unsigned int /* version */) { - ar & BOOST_SERIALIZATION_NVP(ratio); - ar & BOOST_SERIALIZATION_NVP(a); - ar & BOOST_SERIALIZATION_NVP(b); + ar & BOOST_SERIALIZATION_NVP(ratio); + ar & BOOST_SERIALIZATION_NVP(a); + ar & BOOST_SERIALIZATION_NVP(b); } } // namespace ann diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 2bda68c2d8..6b0d8e41d6 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -450,35 +450,34 @@ BOOST_AUTO_TEST_CASE(SimpleAlphaDropoutLayerTest) */ BOOST_AUTO_TEST_CASE(AlphaDropoutProbabilityTest) { - arma::mat input = arma::ones(1500, 1); - const size_t iterations = 10; + arma::mat input = arma::ones(1500, 1); + const size_t iterations = 10; - double probability[5] = { 0.1, 0.3, 0.4, 0.7, 0.8 }; - for (size_t trial = 0; trial < 5; ++trial) + double probability[5] = { 0.1, 0.3, 0.4, 0.7, 0.8 }; + for (size_t trial = 0; trial < 5; ++trial) + { + double nonzeroCount = 0; + for (size_t i = 0; i < iterations; ++i) { - double nonzeroCount = 0; - for (size_t i = 0; i < iterations; ++i) - { - alphaDropout<> module(probability[trial]); - module.Deterministic() = false; + alphaDropout<> module(probability[trial]); + module.Deterministic() = false; - arma::mat output; - module.Forward(std::move(input), std::move(output)); + arma::mat output; + module.Forward(std::move(input), std::move(output)); - // Return a column vector containing the indices of elements of X - // that are not alpha_dash, we just need the number of - // non_alpha_dash values. - arma::uvec non_alpha_dash = arma::find(module.Mask()); - nonzeroCount += non_alpha_dash.n_elem; - } - - const double expected = input.n_elem * (1-probability[trial]) * - iterations; - - const double error = fabs(nonzeroCount - expected) / expected; - - BOOST_REQUIRE_LE(error, 0.15); + // Return a column vector containing the indices of elements of X + // that are not alpha_dash, we just need the number of + // non_alpha_dash values. + arma::uvec non_alpha_dash = arma::find(module.Mask()); + nonzeroCount += non_alpha_dash.n_elem; } + + const double expected = input.n_elem * (1-probability[trial]) * iterations; + + const double error = fabs(nonzeroCount - expected) / expected; + + BOOST_REQUIRE_LE(error, 0.15); + } } /* @@ -486,14 +485,14 @@ BOOST_AUTO_TEST_CASE(AlphaDropoutProbabilityTest) */ BOOST_AUTO_TEST_CASE(NoAlphaDropoutTest) { - arma::mat input = arma::ones(1500, 1); - alphaDropout<> module(0); - module.Deterministic() = false; + arma::mat input = arma::ones(1500, 1); + alphaDropout<> module(0); + module.Deterministic() = false; - arma::mat output; - module.Forward(std::move(input), std::move(output)); + arma::mat output; + module.Forward(std::move(input), std::move(output)); - BOOST_REQUIRE_EQUAL(arma::accu(output), arma::accu(input)); + BOOST_REQUIRE_EQUAL(arma::accu(output), arma::accu(input)); } /**