From bc1dff58d5cf3bef4ec585e3c25101943890f41c Mon Sep 17 00:00:00 2001 From: Mayank Raj Date: Wed, 24 May 2023 08:18:45 +0530 Subject: [PATCH 1/2] Implemented FTSwish Activation Function --- src/mlpack/methods/ann/layer/ftswish.hpp | 114 +++++++++++++++++ src/mlpack/methods/ann/layer/ftswish_impl.hpp | 119 ++++++++++++++++++ src/mlpack/methods/ann/layer/layer_types.hpp | 1 + .../methods/ann/layer/serialization.hpp | 1 + src/mlpack/tests/ann/layer/ftswish.cpp | 62 +++++++++ src/mlpack/tests/ann/layer_test.cpp | 1 + 6 files changed, 298 insertions(+) create mode 100644 src/mlpack/methods/ann/layer/ftswish.hpp create mode 100644 src/mlpack/methods/ann/layer/ftswish_impl.hpp create mode 100644 src/mlpack/tests/ann/layer/ftswish.cpp diff --git a/src/mlpack/methods/ann/layer/ftswish.hpp b/src/mlpack/methods/ann/layer/ftswish.hpp new file mode 100644 index 0000000000..724ee861ae --- /dev/null +++ b/src/mlpack/methods/ann/layer/ftswish.hpp @@ -0,0 +1,114 @@ +/** + * @file methods/ann/layer/flatten_t_swish.hpp + * @author Mayank Raj + * + * Definition of Flatten T Swish layer first introduced in the acoustic model, + * Hock Hung Chieng, Noorhaniza Wahid, Pauline Ong, Sai Raj Kishore Perla, + * "Flatten-T Swish: a thresholded ReLU-Swish-like activation function for deep learning", 2018 + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_FTSWISH_HPP +#define MLPACK_METHODS_ANN_LAYER_FTSWISH_HPP + +#include + +#include "layer.hpp" + +namespace mlpack { + +/** + * The Flatten T Swish activation function, defined by + * + * @f{eqnarray*}{ + * f'(x) &=& \left\{ + * \begin{array}{lr} + * frac{x}{1+exp(-x)} + T & : x \ge 0 \\ + * T & : x < 0 + * \end{array} + * \right. \\ + * f'(x) &=& \left\{ + * \begin{array}{lr} + * \sigma(x)(1 - f(x)) + f(x) & : x > 0 \\ + * 0 & : x \le 0 + * \end{array} + * \right. + * @f} + * + * @tparam MatType Matrix representation to accept as input and use for + * computation. + */ +template +class FTSwishType : public Layer +{ + public: + /** + * Create the Flatten T Swish object using the specified parameters. + * The thresholded value T can be adjusted via T paramaters. + * When the x is < 0, T will be used instead of 0. + * The default value of T is -0.20 as suggested in the paper. + * @param T + */ + FTSwishType(const double T = -0.20); + + //! Clone the FTSwishType object. This handles polymorphism correctly. + FTSwishType* Clone() const { return new FTSwishType(*this); } + + // Virtual destructor. + virtual ~FTSwishType() { } + + //! Copy the given FTSwishType. + FTSwishType(const FTSwishType& other); + //! Take ownership of the given FTSwishType. + FTSwishType(FTSwishType&& other); + //! Copy the given FTSwishType. + FTSwishType& operator=(const FTSwishType& other); + //! Take ownership of the given FTSwishType. + FTSwishType& operator=(FTSwishType&& other); + + /** + * Ordinary feed forward pass of a neural network, evaluating the function + * f(x) by propagating the activity forward through f. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + void Forward(const MatType& input, MatType& output); + + /** + * Ordinary feed backward pass of a neural network, calculating the function + * f(x) by propagating x backwards through f. Using the results from the feed + * forward pass. + * + * @param input The propagated input activation. + * @param gy The backpropagated error. + * @param g The calculated gradient. + */ + void Backward(const MatType& input, const MatType& gy, MatType& g); + + //! Get the threshold value. + double const& Threshold() const { return T; } + //! Modify the threshold value. + double& Threshold() { return T; } + + //! Serialize the layer. + template + void serialize(Archive& ar, const uint32_t /* version */); + + private: + //! Threshold value for x < 0. + double T; +}; // class FTSwishType + +// Convenience typedefs. +typedef FTSwishType FTSwish; + +} // namespace mlpack + +// Include implementation. +#include "ftswish_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/layer/ftswish_impl.hpp b/src/mlpack/methods/ann/layer/ftswish_impl.hpp new file mode 100644 index 0000000000..7619eee03b --- /dev/null +++ b/src/mlpack/methods/ann/layer/ftswish_impl.hpp @@ -0,0 +1,119 @@ +/** + * @file methods/ann/layer/ftswish_impl.hpp + * @author Mayank Raj + * + * Definition of Flatten T Swish layer first introduced in the acoustic model, + * Hock Hung Chieng, Noorhaniza Wahid, Pauline Ong, Sai Raj Kishore Perla, + * + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_FTSWISH_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_FTSWISH_IMPL_HPP + +// In case it hasn't yet been included. +#include "ftswish.hpp" + +namespace mlpack { + +template +FTSwishType::FTSwishType(const double T) : + Layer(), + T(T) +{ + // Nothing to do here. +} + +template +FTSwishType::FTSwishType(const FTSwishType& other) : + Layer(other), + T(other.T) +{ + // Nothing to do. +} + +template +FTSwishType::FTSwishType(FTSwishType&& other) : + Layer(std::move(other)), + T(std::move(other.T)) +{ + // Nothing to do. +} + +template +FTSwishType& +FTSwishType::operator=(const FTSwishType& other) +{ + if (&other != this) + { + Layer::operator=(other); + T = other.T; + } + + return *this; +} + +template +FTSwishType& +FTSwishType::operator=(FTSwishType&& other) +{ + if (&other != this) + { + Layer::operator=(std::move(other)); + T = std::move(other.T); + } + + return *this; +} + +template +void FTSwishType::Forward(const MatType& input, MatType& output) +{ + #pragma omp for + for (size_t i = 0; i < (size_t) input.n_elem; ++i) + { + if (input(i) >= 0) + output(i) = input(i) / (1 + std::exp(-input(i))) + T; + else + output(i) = T; + } +} + +template +void FTSwishType::Backward( + const MatType& input, const MatType& gy, MatType& g) +{ + #pragma omp for + for (size_t i = 0; i < (size_t) input.n_elem; ++i) + { + if (input(i) >= 0) + { + const double f_x = input(i) / (1 + std::exp(-input(i))); + const double sigmoid_x = 1 / (1 + std::exp(-input(i))); + + g(i) = gy(i) * (sigmoid_x * (1 - f_x) + f_x); + } + else + { + g(i) = 0; + } + } +} + +template +template +void FTSwishType::serialize( + Archive& ar, + const uint32_t /* version */) +{ + ar(cereal::base_class>(this)); + + ar(CEREAL_NVP(T)); +} + +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 5d348a14b1..30b63b48ea 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -48,6 +48,7 @@ #include #include #include +#include // Convolution modes. #include diff --git a/src/mlpack/methods/ann/layer/serialization.hpp b/src/mlpack/methods/ann/layer/serialization.hpp index c8daf52a71..dbbd305054 100644 --- a/src/mlpack/methods/ann/layer/serialization.hpp +++ b/src/mlpack/methods/ann/layer/serialization.hpp @@ -70,6 +70,7 @@ CEREAL_REGISTER_TYPE(mlpack::SoftmaxType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::SoftminType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::HardTanHType<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::FTSwishType<__VA_ARGS__>); \ CEREAL_REGISTER_MLPACK_LAYERS(arma::mat); diff --git a/src/mlpack/tests/ann/layer/ftswish.cpp b/src/mlpack/tests/ann/layer/ftswish.cpp new file mode 100644 index 0000000000..52c1cb0a6e --- /dev/null +++ b/src/mlpack/tests/ann/layer/ftswish.cpp @@ -0,0 +1,62 @@ +/* + * @file tests/ann/layer/hard_tanh.cpp + * @author Mayank Raj + * + * Tests the FTSwish layer. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#include +#include + +#include "../../test_catch_tools.hpp" +#include "../../catch.hpp" +#include "../../serialization.hpp" +#include "../ann_test_tools.hpp" + +using namespace mlpack; + +/** + * Simple test case for the FTSwish layer. + */ +TEST_CASE("FTSwishTest", "[ANNLayerTest]") +{ + // Set the threshold value for the FTSwish layer. + double threshold = -0.2; + + // Create the FTSwish layer. + FTSwishType<> layer(threshold); + + // Input and output matrices. + arma::mat input = {{0.234, 1.23, -1.34}, + {1.45, 2.001, -0.98}, + {-3.14, 3.43, 9.9}}; + arma::mat actualOutput = {{-0.06937312, 0.75179685, -0.2}, + {0.97449773, 1.56268497, -0.2}, + {-0.2, 3.1223977, 9.6995033 }}; + arma::mat output; + output.set_size(3,3); + // Forward pass. + layer.Forward(input, output); + + // Test the Forward function + REQUIRE(abs(arma::accu(output - actualOutput)) <= 0.0001); + + arma::mat delta = {{0.0 ,0.84327731, 0.0}, + {0.91985943, 1.05058055 ,0.0 }, + {0.0, 1.08399125 ,1.00053333}}; + + arma::mat gy, g; + gy.set_size(3,3); + gy.fill(1); + g.set_size(3,3); + // Backward pass. + layer.Backward(output, gy, g); + + //Test the Backward function + REQUIRE(abs(arma::accu(g - delta)) <= 0.0001); +} \ No newline at end of file diff --git a/src/mlpack/tests/ann/layer_test.cpp b/src/mlpack/tests/ann/layer_test.cpp index 32bb25425b..1bc6ac3c1e 100644 --- a/src/mlpack/tests/ann/layer_test.cpp +++ b/src/mlpack/tests/ann/layer_test.cpp @@ -37,3 +37,4 @@ #include "layer/parametric_relu.cpp" #include "layer/softmax.cpp" #include "layer/softmin.cpp" +#include "layer/ftswish.cpp" From 94da130f72cd6f09e55603640db25b9743402879 Mon Sep 17 00:00:00 2001 From: Mayank Raj Date: Sun, 28 May 2023 05:25:05 +0530 Subject: [PATCH 2/2] used camel case --- src/mlpack/methods/ann/layer/ftswish_impl.hpp | 9 +++++---- src/mlpack/tests/ann/layer/ftswish.cpp | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/ann/layer/ftswish_impl.hpp b/src/mlpack/methods/ann/layer/ftswish_impl.hpp index 7619eee03b..1298265c68 100644 --- a/src/mlpack/methods/ann/layer/ftswish_impl.hpp +++ b/src/mlpack/methods/ann/layer/ftswish_impl.hpp @@ -91,10 +91,11 @@ void FTSwishType::Backward( { if (input(i) >= 0) { - const double f_x = input(i) / (1 + std::exp(-input(i))); - const double sigmoid_x = 1 / (1 + std::exp(-input(i))); - - g(i) = gy(i) * (sigmoid_x * (1 - f_x) + f_x); + const double fX = input(i) / (1 + std::exp(-input(i))); + const double sigmoidX = 1 / (1 + std::exp(-input(i))); + + g(i) = gy(i) * (sigmoidX * (1 - fX) + fX); + } else { diff --git a/src/mlpack/tests/ann/layer/ftswish.cpp b/src/mlpack/tests/ann/layer/ftswish.cpp index 52c1cb0a6e..a1f1ebd31b 100644 --- a/src/mlpack/tests/ann/layer/ftswish.cpp +++ b/src/mlpack/tests/ann/layer/ftswish.cpp @@ -1,5 +1,5 @@ /* - * @file tests/ann/layer/hard_tanh.cpp + * @file tests/ann/layer/ftswish.cpp * @author Mayank Raj * * Tests the FTSwish layer.