apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3454 from mrdaybird/adapt_hard_tanh

Browse Source 
Updated hard_tanh and added to layer_types
This commit is contained in:
Shubham Agrawal
2023-04-04 22:38:58 +05:30
committed by GitHub
parent 5453cb8032 2d3f021af4
commit d58d9c7493
8 changed files with 202 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
HISTORY.md
+2
View File
@@ -1,6 +1,8 @@
### mlpack ?.?.?
###### ????-??-??
* Adapt HardTanH layer (#3454).
* Adapt Softmin layer for new neural network API (#3437).
* Adapt PReLU layer for new neural network API (#3420).
src/mlpack/methods/ann/layer/not_adapted/hard_tanh.hpp → src/mlpack/methods/ann/layer/hard_tanh.hpp
+20 -5
View File
@@ -1,6 +1,7 @@
/**
* @file methods/ann/layer/hard_tanh.hpp
* @author Dhawal Arora
* @author Vaibhav Pathak
*
* Definition and implementation of the HardTanH layer.
*
@@ -44,8 +45,8 @@ namespace mlpack {
* to also be in this type. The type also allows the computation and weight
* type to differ from the input type (Default: arma::mat).
*/
template <typename InputType = arma::mat, typename OutputType = arma::mat>
class HardTanHType : public Layer<InputType, OutputType>
template <typename MatType = arma::mat>
class HardTanHType : public Layer<MatType>
{
public:
/**
@@ -57,6 +58,20 @@ class HardTanHType : public Layer<InputType, OutputType>
* @param minValue Range of the linear region minimum value.
*/
HardTanHType(const double maxValue = 1, const double minValue = -1);
virtual ~HardTanHType() { }
//! Copy the other HardTanH layer
HardTanHType(const HardTanHType& layer);
//! Take ownership of the members of the other HardTanH Layer
HardTanHType(HardTanHType&& layer);
//! Copy the other HardTanH layer
HardTanHType& operator=(const HardTanHType& layer);
//! Take ownership of the members of the other HardTanH Layer
HardTanHType& operator=(HardTanHType&& layer);
//! Clone the HardTanHType object. This handles polymorphism correctly.
HardTanHType* Clone() const { return new HardTanHType(*this); }
@@ -68,7 +83,7 @@ class HardTanHType : public Layer<InputType, OutputType>
* @param input Input data used for evaluating the specified function.
* @param output Resulting output activation.
*/
void Forward(const InputType& input, OutputType& output);
void Forward(const MatType& input, MatType& output);
/**
* Ordinary feed backward pass of a neural network, calculating the function
@@ -79,7 +94,7 @@ class HardTanHType : public Layer<InputType, OutputType>
* @param gy The backpropagated error.
* @param g The calculated gradient.
*/
void Backward(const InputType& input, const OutputType& gy, OutputType& g);
void Backward(const MatType& input, const MatType& gy, MatType& g);
//! Get the maximum value.
double const& MaxValue() const { return maxValue; }
@@ -108,7 +123,7 @@ class HardTanHType : public Layer<InputType, OutputType>
// Convenience typedefs.
// Standard HardTanH layer.
typedef HardTanHType<arma::mat, arma::mat> HardTanH;
typedef HardTanHType<arma::mat> HardTanH;
} // namespace mlpack
src/mlpack/methods/ann/layer/hard_tanh_impl.hpp
+119
View File
@@ -0,0 +1,119 @@
/**
* @file methods/ann/layer/hard_tanh_impl.hpp
* @author Dhawal Arora
* @author Vaibhav Pathak
*
* Implementation and implementation of the HardTanH layer.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_METHODS_ANN_LAYER_HARD_TANH_IMPL_HPP
#define MLPACK_METHODS_ANN_LAYER_HARD_TANH_IMPL_HPP
// In case it hasn't yet been included.
#include "hard_tanh.hpp"
namespace mlpack {
template<typename MatType>
HardTanHType<MatType>::HardTanHType(
const double maxValue,
const double minValue) :
Layer<MatType>(),
maxValue(maxValue),
minValue(minValue)
{
// Nothing to do here.
}
template<typename MatType>
HardTanHType<MatType>::HardTanHType(const HardTanHType& layer) :
Layer<MatType>(layer),
maxValue(layer.maxValue),
minValue(layer.minValue)
{
// Nothing to do here.
}
template<typename MatType>
HardTanHType<MatType>::HardTanHType(HardTanHType&& layer) :
Layer<MatType>(std::move(layer)),
maxValue(std::move(layer.maxValue)),
minValue(std::move(layer.minValue))
{
// Nothing to do here.
}
template<typename MatType>
HardTanHType<MatType>& HardTanHType<MatType>::operator=(const HardTanHType& layer)
{
if (&layer != this)
{
Layer<MatType>::operator=(layer);
maxValue = layer.maxValue;
minValue = layer.minValue;
}
return *this;
}
template<typename MatType>
HardTanHType<MatType>& HardTanHType<MatType>::operator=(HardTanHType&& layer)
{
if (&layer != this)
{
Layer<MatType>::operator=(std::move(layer));
maxValue = std::move(layer.maxValue);
minValue = std::move(layer.minValue);
}
return *this;
}
template<typename MatType>
void HardTanHType<MatType>::Forward(
const MatType& input, MatType& output)
{
#pragma omp parallel for
for (size_t i = 0; i < input.n_elem; ++i)
{
output(i) = (input(i) > maxValue ? maxValue :
(input(i) < minValue ? minValue : input(i)));
}
}
template<typename MatType>
void HardTanHType<MatType>::Backward(
const MatType& input, const MatType& gy, MatType& g)
{
g = gy;
#pragma omp parallel for
for (size_t i = 0; i < input.n_elem; ++i)
{
// input should not have any values greater than maxValue
// and lesser than minValue
if (input(i) <= minValue || input(i) >= maxValue)
{
g(i) = 0;
}
}
}
template<typename MatType>
template<typename Archive>
void HardTanHType<MatType>::serialize(
Archive& ar,
const uint32_t /* version */)
{
ar(cereal::base_class<Layer<MatType>>(this));
ar(CEREAL_NVP(maxValue));
ar(CEREAL_NVP(minValue));
}
} // namespace mlpack
#endif
src/mlpack/methods/ann/layer/layer_types.hpp
+1
View File
@@ -32,6 +32,7 @@
#include <mlpack/methods/ann/layer/dropout.hpp>
#include <mlpack/methods/ann/layer/elu.hpp>
#include <mlpack/methods/ann/layer/grouped_convolution.hpp>
#include <mlpack/methods/ann/layer/hard_tanh.hpp>
#include <mlpack/methods/ann/layer/identity.hpp>
#include <mlpack/methods/ann/layer/leaky_relu.hpp>
#include <mlpack/methods/ann/layer/linear.hpp>
src/mlpack/methods/ann/layer/not_adapted/hard_tanh_impl.hpp
-69
View File
@@ -1,69 +0,0 @@
/**
* @file methods/ann/layer/hard_tanh_impl.hpp
* @author Dhawal Arora
*
* Implementation and implementation of the HardTanH layer.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_METHODS_ANN_LAYER_HARD_TANH_IMPL_HPP
#define MLPACK_METHODS_ANN_LAYER_HARD_TANH_IMPL_HPP
// In case it hasn't yet been included.
#include "hard_tanh.hpp"
namespace mlpack {
template<typename InputType, typename OutputType>
HardTanHType<InputType, OutputType>::HardTanHType(
const double maxValue,
const double minValue) :
maxValue(maxValue),
minValue(minValue)
{
// Nothing to do here.
}
template<typename InputType, typename OutputType>
void HardTanHType<InputType, OutputType>::Forward(
const InputType& input, OutputType& output)
{
for (size_t i = 0; i < input.n_elem; ++i)
{
output(i) = (output(i) > maxValue ? maxValue :
(output(i) < minValue ? minValue : output(i)));
}
}
template<typename InputType, typename OutputType>
void HardTanHType<InputType, OutputType>::Backward(
const InputType& input, const OutputType& gy, OutputType& g)
{
g = gy;
for (size_t i = 0; i < input.n_elem; ++i)
{
if (input(i) < minValue || input(i) > maxValue)
{
g(i) = 0;
}
}
}
template<typename InputType, typename OutputType>
template<typename Archive>
void HardTanHType<InputType, OutputType>::serialize(
Archive& ar,
const uint32_t /* version */)
{
ar(cereal::base_class<Layer<InputType, OutputType>>(this));
ar(CEREAL_NVP(maxValue));
ar(CEREAL_NVP(minValue));
}
} // namespace mlpack
#endif
src/mlpack/methods/ann/layer/serialization.hpp
+1
View File
@@ -69,6 +69,7 @@
CEREAL_REGISTER_TYPE(mlpack::RBFType<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::SoftmaxType<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::SoftminType<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::HardTanHType<__VA_ARGS__>); \
CEREAL_REGISTER_MLPACK_LAYERS(arma::mat);
src/mlpack/tests/ann/layer/hard_tanh.cpp
+58
View File
@@ -0,0 +1,58 @@
/*
* @file tests/ann/layer/hard_tanh.cpp
* @author Vaibhav Pathak
*
* Tests the hard_tanh layer
*
*/
#include <mlpack/core.hpp>
#include <mlpack/methods/ann.hpp>
#include "../../test_catch_tools.hpp"
#include "../../catch.hpp"
#include "../../serialization.hpp"
#include "../ann_test_tools.hpp"
using namespace mlpack;
/**
* Simple HardTanH module test
*/
TEST_CASE("SimpleHardTanHTest", "[ANNLayerTest]")
{
arma::mat output, gy, g;
arma::mat input = {{-1.3743, -0.5565, 0.2742, -0.0151, -1.4871},
{1.5797, -4.2711, -2.2505, -1.7105, -1.2544},
{0.4023, 0.5676, 2.3100, 1.6658, -0.1907},
{0.1897, 0.9097, 0.1418, -1.5349, 0.1225},
{-0.1101, -3.3656, -5.4033, -2.2240, -3.3235}};
arma::mat actualOutput = {{-1.0000, -0.5565, 0.2742, -0.0151, -1.0000},
{1.0000, -1.0000, -1.0000, -1.0000, -1.0000},
{0.4023, 0.5676, 1.0000, 1.0000, -0.1907},
{0.1897, 0.9097, 0.1418, -1.0000, 0.1225},
{-0.1101, -1.0000, -1.0000, -1.0000, -1.0000}};
HardTanH module;
output.set_size(5,5);
// Test the Forward function
module.Forward(input, output);
REQUIRE(arma::accu(output - actualOutput) == Approx(0).epsilon(1e-4));
arma::mat delta = {{0 , 1.0, 1.0, 1.0, 0.0},
{0 , 0 , 0 , 0.0, 0.0},
{1.0, 1.0, 0 , 0.0, 1.0},
{1.0, 1.0, 1.0, 0.0, 1.0},
{1.0, 0 , 0.0, 0.0, 0.0}};
gy.set_size(5,5);
gy.fill(1);
g.set_size(5,5);
//Test the Backward function
module.Backward(output, gy, g);
REQUIRE(arma::accu(g - delta) == Approx(0).epsilon(1e-4));
}
src/mlpack/tests/ann/layer_test.cpp
+1
View File
@@ -26,6 +26,7 @@
#include "layer/concatenate.cpp"
#include "layer/dropout.cpp"
#include "layer/grouped_convolution.cpp"
#include "layer/hard_tanh.cpp"
#include "layer/identity.cpp"
#include "layer/linear3d.cpp"
#include "layer/linear_no_bias.cpp"