From 8bb8ea1bad84b3fcc6ae6e22c3970e4c5b44ad39 Mon Sep 17 00:00:00 2001 From: Utkarsh Rai Date: Sat, 3 Oct 2020 01:22:46 +0530 Subject: [PATCH 1/4] Added Weight Size to concat and fast_lstm layers and renamed parameters to weights in dropconnect. --- src/mlpack/methods/ann/layer/concat.hpp | 14 ++++++++++---- src/mlpack/methods/ann/layer/concat_impl.hpp | 4 ++-- src/mlpack/methods/ann/layer/dropconnect.hpp | 6 +++--- src/mlpack/methods/ann/layer/fast_lstm.hpp | 6 ++++++ 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/mlpack/methods/ann/layer/concat.hpp b/src/mlpack/methods/ann/layer/concat.hpp index 30bfe4f3b5..bb0d733651 100644 --- a/src/mlpack/methods/ann/layer/concat.hpp +++ b/src/mlpack/methods/ann/layer/concat.hpp @@ -165,9 +165,9 @@ class Concat } //! Return the initial point for the optimization. - const arma::mat& Parameters() const { return parameters; } + const arma::mat& Parameters() const { return weights; } //! Modify the initial point for the optimization. - arma::mat& Parameters() { return parameters; } + arma::mat& Parameters() { return weights; } //! Get the value of run parameter. bool Run() const { return run; } @@ -196,6 +196,12 @@ class Concat //! Get the axis of concatenation. size_t const& ConcatAxis() const { return axis; } + //! Get the size of the weight matrix. + size_t WeightSize() const + { + return 0; + } + /** * Serialize the layer */ @@ -225,8 +231,8 @@ class Concat //! Locally-stored network modules. std::vector > network; - //! Locally-stored model parameters. - arma::mat parameters; + //! Locally-stored model weights. + OutputDataType weights; //! Locally-stored delta visitor. DeltaVisitor deltaVisitor; diff --git a/src/mlpack/methods/ann/layer/concat_impl.hpp b/src/mlpack/methods/ann/layer/concat_impl.hpp index a75fea8697..62de73a74e 100644 --- a/src/mlpack/methods/ann/layer/concat_impl.hpp +++ b/src/mlpack/methods/ann/layer/concat_impl.hpp @@ -33,7 +33,7 @@ Concat::Concat( run(run), channels(1) { - parameters.set_size(0, 0); + weights.set_size(0, 0); } template::Concat( model(model), run(run) { - parameters.set_size(0, 0); + weights.set_size(0, 0); // Parameters to help calculate the number of channels. size_t oldColSize = 1, newColSize = 1; diff --git a/src/mlpack/methods/ann/layer/dropconnect.hpp b/src/mlpack/methods/ann/layer/dropconnect.hpp index a1411845a4..cf96d6237f 100644 --- a/src/mlpack/methods/ann/layer/dropconnect.hpp +++ b/src/mlpack/methods/ann/layer/dropconnect.hpp @@ -115,9 +115,9 @@ class DropConnect std::vector >& Model() { return network; } //! Get the parameters. - OutputDataType const& Parameters() const { return parameters; } + OutputDataType const& Parameters() const { return weights; } //! Modify the parameters. - OutputDataType& Parameters() { return parameters; } + OutputDataType& Parameters() { return weights; } //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } @@ -164,7 +164,7 @@ class DropConnect double scale; //! Locally-stored weight object. - OutputDataType parameters; + OutputDataType weights; //! Locally-stored delta object. OutputDataType delta; diff --git a/src/mlpack/methods/ann/layer/fast_lstm.hpp b/src/mlpack/methods/ann/layer/fast_lstm.hpp index b934c0c5ea..eb5d2ef855 100644 --- a/src/mlpack/methods/ann/layer/fast_lstm.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm.hpp @@ -164,6 +164,12 @@ class FastLSTM //! Get the number of output units. size_t OutSize() const { return outSize; } + //! Get the size of the weight matrix. + size_t WeightSize() const + { + return 4 * outSize * inSize + 4 * outSize + 4 * outSize * outSize; + } + /** * Serialize the layer */ From 7ef7ec02a5faaca5452d7df2a05cb769fcb0d24e Mon Sep 17 00:00:00 2001 From: Utkarsh Rai Date: Fri, 9 Oct 2020 10:53:12 +0530 Subject: [PATCH 2/4] Replaced the expression with WeightSize().) Added tests. --- src/mlpack/methods/ann/layer/add_impl.hpp | 2 +- .../ann/layer/atrous_convolution_impl.hpp | 3 +- src/mlpack/methods/ann/layer/concat.hpp | 5 +- .../methods/ann/layer/fast_lstm_impl.hpp | 3 +- src/mlpack/methods/ann/layer/linear_impl.hpp | 2 +- src/mlpack/tests/ann_visitor_test.cpp | 75 +++++++++++++++++-- 6 files changed, 73 insertions(+), 17 deletions(-) diff --git a/src/mlpack/methods/ann/layer/add_impl.hpp b/src/mlpack/methods/ann/layer/add_impl.hpp index 79f2a4386f..4fc8f42d0b 100644 --- a/src/mlpack/methods/ann/layer/add_impl.hpp +++ b/src/mlpack/methods/ann/layer/add_impl.hpp @@ -23,7 +23,7 @@ template Add::Add(const size_t outSize) : outSize(outSize) { - weights.set_size(outSize, 1); + weights.set_size(WeightSize(), 1); } template diff --git a/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp b/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp index 3536690037..7d6534a18e 100644 --- a/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp @@ -122,8 +122,7 @@ AtrousConvolution< dilationWidth(dilationWidth), dilationHeight(dilationHeight) { - weights.set_size((outSize * inSize * kernelWidth * kernelHeight) + outSize, - 1); + weights.set_size(WeightSize(), 1); // Transform paddingType to lowercase. std::string paddingTypeLow = paddingType; diff --git a/src/mlpack/methods/ann/layer/concat.hpp b/src/mlpack/methods/ann/layer/concat.hpp index bb0d733651..c6e5937167 100644 --- a/src/mlpack/methods/ann/layer/concat.hpp +++ b/src/mlpack/methods/ann/layer/concat.hpp @@ -197,10 +197,7 @@ class Concat size_t const& ConcatAxis() const { return axis; } //! Get the size of the weight matrix. - size_t WeightSize() const - { - return 0; - } + size_t WeightSize() const { return 0; } /** * Serialize the layer diff --git a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp index 24b3544fd1..b350a704b3 100644 --- a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp @@ -42,8 +42,7 @@ FastLSTM::FastLSTM( { // Weights for: input to gate layer (4 * outsize * inSize + 4 * outsize) // and output to gate (4 * outSize). - weights.set_size( - 4 * outSize * inSize + 4 * outSize + 4 * outSize * outSize, 1); + weights.set_size(WeightSize(), 1); } template diff --git a/src/mlpack/methods/ann/layer/linear_impl.hpp b/src/mlpack/methods/ann/layer/linear_impl.hpp index 79799fea98..875147f7b7 100644 --- a/src/mlpack/methods/ann/layer/linear_impl.hpp +++ b/src/mlpack/methods/ann/layer/linear_impl.hpp @@ -38,7 +38,7 @@ Linear::Linear( outSize(outSize), regularizer(regularizer) { - weights.set_size(outSize * inSize + outSize, 1); + weights.set_size(WeightSize(), 1); } template linear = new Linear<>(randomSize, randomSize); + LayerTypes<> linearLayer = new Linear<>(randomInSize, randomOutSize); - size_t weightSize = boost::apply_visitor(WeightSizeVisitor(), - linear); + size_t weightSize = boost::apply_visitor(WeightSizeVisitor(), linearLayer); - REQUIRE(weightSize == randomSize * randomSize + randomSize); + REQUIRE(weightSize == randomInSize * randomOutSize + randomOutSize); } +/** + * Test that WeightSizeVisitor works properly for concat layer. + */ +TEST_CASE("WeightSizeVisitorTestForConcatLayer", "[ANNVisitorTest]") +{ + LayerTypes<> concatLayer = new Concat<>(); + + size_t weightSize = boost::apply_visitor(WeightSizeVisitor(), concatLayer); + + REQUIRE(weightSize == 0); +} + +/** + * Test that WeightSizeVisitor works properly for fast lstm layer. + */ +TEST_CASE("WeightSizeVisitorTestForFastLSTMLayer", "[ANNVisitorTest]") +{ + size_t randomInSize = arma::randi(arma::distr_param(1, 100)); + size_t randomOutSize = arma::randi(arma::distr_param(1, 100)); + + LayerTypes<> fastLSTMLayer = new FastLSTM<>(randomInSize, randomOutSize); + + size_t weightSize = boost::apply_visitor(WeightSizeVisitor(), fastLSTMLayer); + + REQUIRE(weightSize == 4 * randomInSize * randomOutSize + 4 * randomOutSize + + 4 * randomOutSize * randomOutSize); +} + +/** + * Test that WeightSizeVisitor works properly for Add layer. + */ +TEST_CASE("WeightSizeVisitorTestForAddLayer", "[ANNVisitorTest]") +{ + size_t randomOutSize = arma::randi(arma::distr_param(1, 100)); + + LayerTypes<> addLayer = new Add<>(randomOutSize); + + size_t weightSize = boost::apply_visitor(WeightSizeVisitor(), addLayer); + + REQUIRE(weightSize == randomOutSize); +} + +/** + * Test that WeightSizeVisitor works properly for Atrous Convolution Layer. + */ +TEST_CASE("WeightSizeVisitorTestForAtrousConvolutionLayer", "[ANNVisitorTest]") +{ + size_t randomInSize = arma::randi(arma::distr_param(1, 100)); + size_t randomOutSize = arma::randi(arma::distr_param(1, 100)); + size_t randomKernelWidth = arma::randi(arma::distr_param(1, 100)); + size_t randomKernelHeight = arma::randi(arma::distr_param(1, 100)); + + LayerTypes<> atrousConvLayer = new AtrousConvolution<>(randomInSize, randomOutSize, + randomKernelWidth, randomKernelHeight); + + size_t weightSize = boost::apply_visitor(WeightSizeVisitor(), + atrousConvLayer); + + REQUIRE(weightSize == randomOutSize * randomInSize * randomKernelWidth + * randomKernelHeight + randomOutSize); +} From 8f9685a06a81f60fd0c11964af33e1641e05d126 Mon Sep 17 00:00:00 2001 From: Utkarsh Rai Date: Mon, 12 Oct 2020 10:35:14 +0530 Subject: [PATCH 3/4] WeightSize() for dropconnect. --- src/mlpack/methods/ann/layer/dropconnect.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/mlpack/methods/ann/layer/dropconnect.hpp b/src/mlpack/methods/ann/layer/dropconnect.hpp index cf96d6237f..2b07cfed8f 100644 --- a/src/mlpack/methods/ann/layer/dropconnect.hpp +++ b/src/mlpack/methods/ann/layer/dropconnect.hpp @@ -150,6 +150,9 @@ class DropConnect scale = 1.0 / (1.0 - ratio); } + //! Return the size of the weight matrix. + size_t WeightSize() const { return 0; } + /** * Serialize the layer. */ From 8b3b5522494e9de1f4b01613c0351ef928ecebf5 Mon Sep 17 00:00:00 2001 From: Utkarsh Rai Date: Tue, 13 Oct 2020 12:26:10 +0530 Subject: [PATCH 4/4] Indentation fix. --- src/mlpack/tests/ann_visitor_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/ann_visitor_test.cpp b/src/mlpack/tests/ann_visitor_test.cpp index 3849e85740..b8690e6bd6 100644 --- a/src/mlpack/tests/ann_visitor_test.cpp +++ b/src/mlpack/tests/ann_visitor_test.cpp @@ -142,7 +142,7 @@ TEST_CASE("WeightSizeVisitorTestForAtrousConvolutionLayer", "[ANNVisitorTest]") randomKernelWidth, randomKernelHeight); size_t weightSize = boost::apply_visitor(WeightSizeVisitor(), - atrousConvLayer); + atrousConvLayer); REQUIRE(weightSize == randomOutSize * randomInSize * randomKernelWidth * randomKernelHeight + randomOutSize);