From 3d2d71ced56ec6ad69ac8fa5f7ceb67c8c14a1b0 Mon Sep 17 00:00:00 2001 From: walragatver Date: Wed, 7 Aug 2019 00:08:43 +0530 Subject: [PATCH 1/4] add bias_set_visitor --- .../methods/ann/layer/atrous_convolution.hpp | 3 + src/mlpack/methods/ann/layer/convolution.hpp | 3 + src/mlpack/methods/ann/layer/layer_traits.hpp | 4 + src/mlpack/methods/ann/layer/linear.hpp | 3 + .../ann/layer/transposed_convolution.hpp | 3 + src/mlpack/methods/ann/visitor/CMakeLists.txt | 2 + .../methods/ann/visitor/bias_set_visitor.hpp | 80 ++++++++++++++++ .../ann/visitor/bias_set_visitor_impl.hpp | 94 +++++++++++++++++++ 8 files changed, 192 insertions(+) create mode 100644 src/mlpack/methods/ann/visitor/bias_set_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/bias_set_visitor_impl.hpp diff --git a/src/mlpack/methods/ann/layer/atrous_convolution.hpp b/src/mlpack/methods/ann/layer/atrous_convolution.hpp index 52281086cc..608f58a045 100644 --- a/src/mlpack/methods/ann/layer/atrous_convolution.hpp +++ b/src/mlpack/methods/ann/layer/atrous_convolution.hpp @@ -165,6 +165,9 @@ class AtrousConvolution //! Modify the output height. size_t& OutputHeight() { return outputHeight; } + //! Modify the bias weights of the layer. + arma::mat& Bias() { return bias; } + /** * Serialize the layer */ diff --git a/src/mlpack/methods/ann/layer/convolution.hpp b/src/mlpack/methods/ann/layer/convolution.hpp index 83c2e09457..5298a69b03 100644 --- a/src/mlpack/methods/ann/layer/convolution.hpp +++ b/src/mlpack/methods/ann/layer/convolution.hpp @@ -161,6 +161,9 @@ class Convolution //! Modify the output height. size_t& OutputHeight() { return outputHeight; } + //! Modify the bias weights of the layer. + arma::mat& Bias() { return bias; } + /** * Serialize the layer */ diff --git a/src/mlpack/methods/ann/layer/layer_traits.hpp b/src/mlpack/methods/ann/layer/layer_traits.hpp index 31a4687dd8..1955235f87 100644 --- a/src/mlpack/methods/ann/layer/layer_traits.hpp +++ b/src/mlpack/methods/ann/layer/layer_traits.hpp @@ -112,6 +112,10 @@ HAS_MEM_FUNC(Loss, HasLoss); // can use with SFINAE to catch when a type has a Run() function. HAS_MEM_FUNC(Run, HasRunCheck); +// This gives us a HasBiasCheck type (where U is a function pointer) we +// can use with SFINAE to catch when a type has a Bias() function. +HAS_MEM_FUNC(Bias, HasBiasCheck); + } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/linear.hpp b/src/mlpack/methods/ann/layer/linear.hpp index 74c709adfb..c9b73fd427 100644 --- a/src/mlpack/methods/ann/layer/linear.hpp +++ b/src/mlpack/methods/ann/layer/linear.hpp @@ -117,6 +117,9 @@ class Linear //! Modify the gradient. OutputDataType& Gradient() { return gradient; } + //! Modify the bias weights of the layer. + arma::mat& Bias() { return bias; } + /** * Serialize the layer */ diff --git a/src/mlpack/methods/ann/layer/transposed_convolution.hpp b/src/mlpack/methods/ann/layer/transposed_convolution.hpp index 8df31aa491..012833149c 100644 --- a/src/mlpack/methods/ann/layer/transposed_convolution.hpp +++ b/src/mlpack/methods/ann/layer/transposed_convolution.hpp @@ -162,6 +162,9 @@ class TransposedConvolution //! Modify the output height. size_t& OutputHeight() { return outputHeight; } + //! Modify the bias weights of the layer. + arma::mat& Bias() { return bias; } + /** * Serialize the layer */ diff --git a/src/mlpack/methods/ann/visitor/CMakeLists.txt b/src/mlpack/methods/ann/visitor/CMakeLists.txt index 67178361d8..43bcf71225 100644 --- a/src/mlpack/methods/ann/visitor/CMakeLists.txt +++ b/src/mlpack/methods/ann/visitor/CMakeLists.txt @@ -5,6 +5,8 @@ set(SOURCES add_visitor_impl.hpp backward_visitor.hpp backward_visitor_impl.hpp + bias_set_visitor.hpp + bias_set_visitor_impl.hpp copy_visitor.hpp copy_visitor_impl.hpp delete_visitor.hpp diff --git a/src/mlpack/methods/ann/visitor/bias_set_visitor.hpp b/src/mlpack/methods/ann/visitor/bias_set_visitor.hpp new file mode 100644 index 0000000000..e104f3db58 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/bias_set_visitor.hpp @@ -0,0 +1,80 @@ +/** + * @file bias_set_visitor.hpp + * @author Toshal Agrawal + * + * This file provides an abstraction for the Bias() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_BIAS_SET_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_BIAS_SET_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * BiasSetVisitor update the module bias parameters given the parameters set. + */ +class BiasSetVisitor : public boost::static_visitor +{ + public: + //! Update the bias parameters given the parameters set and offset. + BiasSetVisitor(arma::mat&& weight, const size_t offset = 0); + + //! Update the parameters set. + template + size_t operator()(LayerType* layer) const; + + private: + //! The parameters set. + arma::mat&& weight; + + //! The parameters offset. + const size_t offset; + + //! Do not update the bias parameters if the module doesn't implement the + //! Bias() or Model() function. + template + typename std::enable_if< + !HasBiasCheck::value && + !HasModelCheck::value, size_t>::type + LayerSize(T* layer) const; + + //! Update the bias parameters if the module implements the Model() function. + template + typename std::enable_if< + !HasBiasCheck::value && + HasModelCheck::value, size_t>::type + LayerSize(T* layer) const; + + //! Update the bias parameters if the module implements the Bias() function. + template + typename std::enable_if< + HasBiasCheck::value && + !HasModelCheck::value, size_t>::type + LayerSize(T* layer) const; + + //! Update the bias parameters if the module implements the Model() and + //! Bias() function. + template + typename std::enable_if< + HasBiasCheck::value && + HasModelCheck::value, size_t>::type + LayerSize(T* layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "bias_set_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/bias_set_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/bias_set_visitor_impl.hpp new file mode 100644 index 0000000000..c27f1bbe95 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/bias_set_visitor_impl.hpp @@ -0,0 +1,94 @@ +/** + * @file bias_set_visitor_impl.hpp + * @author Toshal Agrawal + * + * Implementation of the Bias() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_BIAS_SET_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_BIAS_SET_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "bias_set_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! BiasSetVisitor visitor class. +inline BiasSetVisitor::BiasSetVisitor(arma::mat&& weight, const size_t offset) : + weight(std::move(weight)), + offset(offset) +{ + /* Nothing to do here. */ +} + +template +inline size_t BiasSetVisitor::operator()(LayerType* layer) const +{ + return LayerSize(layer); +} + +template +inline typename std::enable_if< + !HasBiasCheck::value && + !HasModelCheck::value, size_t>::type +BiasSetVisitor::LayerSize(T* /* layer */) const +{ + return 0; +} + +template +inline typename std::enable_if< + !HasBiasCheck::value && + HasModelCheck::value, size_t>::type +BiasSetVisitor::LayerSize(T* layer) const +{ + size_t modelOffset = 0; + for (size_t i = 0; i < layer->Model().size(); ++i) + { + modelOffset += boost::apply_visitor(BiasSetVisitor( + std::move(weight), modelOffset + offset), layer->Model()[i]); + } + + return modelOffset; +} + +template +inline typename std::enable_if< + HasBiasCheck::value && + !HasModelCheck::value, size_t>::type +BiasSetVisitor::LayerSize(T* layer) const +{ + layer->Bias() = arma::mat(weight.memptr() + offset, + layer->Bias().n_rows, layer->Bias().n_cols, false, false); + + return layer->Bias().n_elem; +} + +template +inline typename std::enable_if< + HasBiasCheck::value && + HasModelCheck::value, size_t>::type +BiasSetVisitor::LayerSize(T* layer) const +{ + layer->Bias() = arma::mat(weight.memptr() + offset, + layer->Bias().n_rows, layer->Bias().n_cols, false, false); + + size_t modelOffset = layer->Bias().n_elem; + for (size_t i = 0; i < layer->Model().size(); ++i) + { + modelOffset += boost::apply_visitor(BiasSetVisitor( + std::move(weight), modelOffset + offset), layer->Model()[i]); + } + + return modelOffset; +} + +} // namespace ann +} // namespace mlpack + +#endif From 54ca394b752cd8133a29cc4f9af7873907d13c0c Mon Sep 17 00:00:00 2001 From: walragatver Date: Mon, 12 Aug 2019 20:23:44 +0530 Subject: [PATCH 2/4] Add test. --- src/mlpack/tests/CMakeLists.txt | 1 + src/mlpack/tests/ann_visitor_test.cpp | 55 +++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 src/mlpack/tests/ann_visitor_test.cpp diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index ba866f0a3e..e1b98d3342 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -8,6 +8,7 @@ add_executable(mlpack_test ann_layer_test.cpp ann_regularizer_test.cpp ann_test_tools.hpp + ann_visitor_test.cpp arma_extend_test.cpp armadillo_svd_test.cpp async_learning_test.cpp diff --git a/src/mlpack/tests/ann_visitor_test.cpp b/src/mlpack/tests/ann_visitor_test.cpp new file mode 100644 index 0000000000..f2c9265a40 --- /dev/null +++ b/src/mlpack/tests/ann_visitor_test.cpp @@ -0,0 +1,55 @@ +/** + * @file ann_visitor_test.cpp + * + * Tests for testing visitors in ANN's of mlpack. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include +#include +#include +#include +#include +#include + +#include +#include "test_tools.hpp" + +using namespace mlpack; +using namespace mlpack::ann; + +BOOST_AUTO_TEST_SUITE(ANNVisitorTest); + +BOOST_AUTO_TEST_CASE(BiasSetVisitorTest) +{ + LayerTypes<> linear = new Linear<>(10, 10); + + arma::mat layerWeights(110, 1); + layerWeights.zeros(); + + ResetVisitor resetVisitor; + + boost::apply_visitor(WeightSetVisitor(std::move(layerWeights), 0), linear); + + boost::apply_visitor(resetVisitor, linear); + + arma::mat weight = {"1 2 3 4 5 6 7 8 9 10"}; + + size_t biasSize = boost::apply_visitor(BiasSetVisitor(std::move(weight), + 0), linear); + + BOOST_REQUIRE_EQUAL(biasSize, 10); + + arma::mat input(10, 1), output; + input.randu(); + + boost::apply_visitor(ForwardVisitor(std::move(input), std::move(output)), + linear); + + BOOST_REQUIRE_EQUAL(arma::accu(output), 55); +} + +BOOST_AUTO_TEST_SUITE_END(); From 6c16e5ec3c6a488470678a9abff9673e780b5e84 Mon Sep 17 00:00:00 2001 From: walragatver Date: Mon, 12 Aug 2019 22:34:49 +0530 Subject: [PATCH 3/4] Add Description. --- src/mlpack/tests/ann_visitor_test.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/mlpack/tests/ann_visitor_test.cpp b/src/mlpack/tests/ann_visitor_test.cpp index f2c9265a40..e7db9708b2 100644 --- a/src/mlpack/tests/ann_visitor_test.cpp +++ b/src/mlpack/tests/ann_visitor_test.cpp @@ -23,6 +23,9 @@ using namespace mlpack::ann; BOOST_AUTO_TEST_SUITE(ANNVisitorTest); +/** + * Test that the BiasSetVisitor works properly. + */ BOOST_AUTO_TEST_CASE(BiasSetVisitorTest) { LayerTypes<> linear = new Linear<>(10, 10); From 8970b593e6e10638b5303eb7578a450f7e113fa8 Mon Sep 17 00:00:00 2001 From: walragatver Date: Wed, 14 Aug 2019 19:04:34 +0530 Subject: [PATCH 4/4] Style Fix. --- src/mlpack/methods/ann/visitor/bias_set_visitor.hpp | 2 +- src/mlpack/methods/ann/visitor/bias_set_visitor_impl.hpp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/visitor/bias_set_visitor.hpp b/src/mlpack/methods/ann/visitor/bias_set_visitor.hpp index e104f3db58..1d7541b67d 100644 --- a/src/mlpack/methods/ann/visitor/bias_set_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/bias_set_visitor.hpp @@ -21,7 +21,7 @@ namespace mlpack { namespace ann { /** - * BiasSetVisitor update the module bias parameters given the parameters set. + * BiasSetVisitor updates the module bias parameters given the parameters set. */ class BiasSetVisitor : public boost::static_visitor { diff --git a/src/mlpack/methods/ann/visitor/bias_set_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/bias_set_visitor_impl.hpp index c27f1bbe95..8f396b54de 100644 --- a/src/mlpack/methods/ann/visitor/bias_set_visitor_impl.hpp +++ b/src/mlpack/methods/ann/visitor/bias_set_visitor_impl.hpp @@ -48,6 +48,7 @@ inline typename std::enable_if< BiasSetVisitor::LayerSize(T* layer) const { size_t modelOffset = 0; + for (size_t i = 0; i < layer->Model().size(); ++i) { modelOffset += boost::apply_visitor(BiasSetVisitor( @@ -79,6 +80,7 @@ BiasSetVisitor::LayerSize(T* layer) const layer->Bias().n_rows, layer->Bias().n_cols, false, false); size_t modelOffset = layer->Bias().n_elem; + for (size_t i = 0; i < layer->Model().size(); ++i) { modelOffset += boost::apply_visitor(BiasSetVisitor(