From 5220de7d143e911bd5dc5c91e78ba80d9c22bdfc Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 28 May 2021 12:52:21 -0400 Subject: [PATCH] Fix some minor compilation issues. --- src/mlpack/methods/ann/layer/add_merge.hpp | 2 +- .../methods/ann/layer/alpha_dropout.hpp | 2 +- .../methods/ann/layer/atrous_convolution.hpp | 31 ++-- .../ann/layer/atrous_convolution_impl.hpp | 7 +- src/mlpack/methods/ann/layer/base_layer.hpp | 169 +++++++----------- src/mlpack/methods/ann/layer/batch_norm.hpp | 2 +- .../methods/ann/layer/concat_performance.hpp | 2 +- .../ann/layer/concat_performance_impl.hpp | 24 +-- src/mlpack/methods/ann/layer/concatenate.hpp | 2 +- src/mlpack/methods/ann/layer/constant.hpp | 15 ++ .../methods/ann/layer/constant_impl.hpp | 64 +++++++ src/mlpack/methods/ann/layer/dropout.hpp | 2 - .../methods/ann/layer/fast_lstm_impl.hpp | 10 +- src/mlpack/methods/ann/layer/gru.hpp | 18 +- src/mlpack/methods/ann/layer/gru_impl.hpp | 22 ++- src/mlpack/methods/ann/layer/highway_impl.hpp | 8 + src/mlpack/methods/ann/layer/layer.hpp | 6 + src/mlpack/methods/ann/layer/layer_types.hpp | 33 +++- src/mlpack/methods/ann/layer/linear.hpp | 2 - src/mlpack/methods/ann/layer/log_softmax.hpp | 2 - src/mlpack/methods/ann/layer/lstm.hpp | 14 +- src/mlpack/methods/ann/layer/lstm_impl.hpp | 38 ++-- src/mlpack/methods/ann/layer/max_pooling.hpp | 4 + src/mlpack/methods/ann/layer/mean_pooling.hpp | 3 + src/mlpack/methods/ann/layer/padding.hpp | 4 +- src/mlpack/methods/ann/layer/recurrent.hpp | 16 +- .../methods/ann/layer/recurrent_attention.hpp | 10 +- .../ann/layer/recurrent_attention_impl.hpp | 10 +- .../methods/ann/layer/recurrent_impl.hpp | 40 ++--- .../methods/ann/layer/serialization.hpp | 48 ++--- .../methods/ann/layer/spatial_dropout.hpp | 2 +- src/mlpack/methods/ann/layer/weight_norm.hpp | 14 ++ .../methods/ann/layer/weight_norm_impl.hpp | 73 ++++++++ src/mlpack/tests/loss_functions_test.cpp | 12 +- 34 files changed, 405 insertions(+), 306 deletions(-) diff --git a/src/mlpack/methods/ann/layer/add_merge.hpp b/src/mlpack/methods/ann/layer/add_merge.hpp index 6012dca63c..c2922fd002 100644 --- a/src/mlpack/methods/ann/layer/add_merge.hpp +++ b/src/mlpack/methods/ann/layer/add_merge.hpp @@ -34,7 +34,7 @@ template< typename InputType = arma::mat, typename OutputType = arma::mat > -class AddMerge +class AddMerge : public Layer { public: /** diff --git a/src/mlpack/methods/ann/layer/alpha_dropout.hpp b/src/mlpack/methods/ann/layer/alpha_dropout.hpp index ea09d20158..aeb98bef15 100644 --- a/src/mlpack/methods/ann/layer/alpha_dropout.hpp +++ b/src/mlpack/methods/ann/layer/alpha_dropout.hpp @@ -89,7 +89,7 @@ class AlphaDropout : public Layer OutputType& Delta() { return delta; } //! The value of the deterministic parameter. - bool Deterministic() const { return deterministic; } + const bool& Deterministic() const { return deterministic; } //! Modify the value of the deterministic parameter. bool& Deterministic() { return deterministic; } diff --git a/src/mlpack/methods/ann/layer/atrous_convolution.hpp b/src/mlpack/methods/ann/layer/atrous_convolution.hpp index 40f488893e..3734482af8 100644 --- a/src/mlpack/methods/ann/layer/atrous_convolution.hpp +++ b/src/mlpack/methods/ann/layer/atrous_convolution.hpp @@ -159,7 +159,6 @@ class AtrousConvolution : public Layer * @param error The calculated error. * @param gradient The calculated gradient. */ - template void Gradient(const InputType& /* input */, const OutputType& error, OutputType& gradient); @@ -198,65 +197,65 @@ class AtrousConvolution : public Layer OutputType& Gradient() { return gradient; } //! Get the input width. - size_t InputWidth() const { return inputWidth; } + const size_t& InputWidth() const { return inputWidth; } //! Modify input the width. size_t& InputWidth() { return inputWidth; } //! Get the input height. - size_t InputHeight() const { return inputHeight; } + const size_t& InputHeight() const { return inputHeight; } //! Modify the input height. size_t& InputHeight() { return inputHeight; } //! Get the output width. - size_t OutputWidth() const { return outputWidth; } + const size_t& OutputWidth() const { return outputWidth; } //! Modify the output width. size_t& OutputWidth() { return outputWidth; } //! Get the output height. - size_t OutputHeight() const { return outputHeight; } + const size_t& OutputHeight() const { return outputHeight; } //! Modify the output height. size_t& OutputHeight() { return outputHeight; } //! Get the input size. - size_t InputSize() const { return inSize; } + const size_t& InputSize() const { return inSize; } //! Get the output size. - size_t OutputSize() const { return outSize; } + const size_t& OutputSize() const { return outSize; } //! Get the kernel width. - size_t KernelWidth() const { return kernelWidth; } + const size_t& KernelWidth() const { return kernelWidth; } //! Modify the kernel width. size_t& KernelWidth() { return kernelWidth; } //! Get the kernel height. - size_t KernelHeight() const { return kernelHeight; } + const size_t& KernelHeight() const { return kernelHeight; } //! Modify the kernel height. size_t& KernelHeight() { return kernelHeight; } //! Get the stride width. - size_t StrideWidth() const { return strideWidth; } + const size_t& StrideWidth() const { return strideWidth; } //! Modify the stride width. size_t& StrideWidth() { return strideWidth; } //! Get the stride height. - size_t StrideHeight() const { return strideHeight; } + const size_t& StrideHeight() const { return strideHeight; } //! Modify the stride height. size_t& StrideHeight() { return strideHeight; } //! Get the dilation rate on the X axis. - size_t DilationWidth() const { return dilationWidth; } + const size_t& DilationWidth() const { return dilationWidth; } //! Modify the dilation rate on the X axis. size_t& DilationWidth() { return dilationWidth; } //! Get the dilation rate on the Y axis. - size_t DilationHeight() const { return dilationHeight; } + const size_t& DilationHeight() const { return dilationHeight; } //! Modify the dilation rate on the Y axis. size_t& DilationHeight() { return dilationHeight; } //! Get the internal Padding layer. - ann::Padding<> const& Padding() const { return padding; } + PaddingType const& Padding() const { return padding; } //! Modify the internal Padding layer. - ann::Padding<>& Padding() { return padding; } + PaddingType& Padding() { return padding; } //! Get size of the weight matrix. size_t WeightSize() const @@ -390,7 +389,7 @@ class AtrousConvolution : public Layer arma::Cube gradientTemp; //! Locally-stored padding layer. - ann::Padding<> padding; + PaddingType padding; //! Locally-stored delta object. OutputType delta; diff --git a/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp b/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp index 8867647856..b9fd3883c6 100644 --- a/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp @@ -144,7 +144,8 @@ AtrousConvolution< InitializeSamePadding(padWLeft, padWRight, padHTop, padHBottom); } - padding = ann::Padding<>(padWLeft, padWRight, padHTop, padHBottom); + padding = PaddingType(padWLeft, padWRight, padHTop, + padHBottom); } template< @@ -329,7 +330,7 @@ void AtrousConvolution< ((OutputType&) error).memptr(), outputWidth, outputHeight, outSize * batchSize, false, false); arma::Cube inputTemp( - const_cast&>(input).memptr(), inputWidth, inputHeight, + const_cast(input).memptr(), inputWidth, inputHeight, inSize * batchSize, false, false); gradient.set_size(weights.n_elem, 1); @@ -417,7 +418,7 @@ void AtrousConvolution< OutputType >::serialize(Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); + ar(cereal::base_class>(this)); ar(CEREAL_NVP(inSize)); ar(CEREAL_NVP(outSize)); diff --git a/src/mlpack/methods/ann/layer/base_layer.hpp b/src/mlpack/methods/ann/layer/base_layer.hpp index 15454aea23..b4cb6417c5 100644 --- a/src/mlpack/methods/ann/layer/base_layer.hpp +++ b/src/mlpack/methods/ann/layer/base_layer.hpp @@ -139,160 +139,123 @@ class BaseLayer : public Layer /** * Standard Sigmoid-Layer using the logistic activation function. */ -// template < -// class ActivationFunction = LogisticFunction, -// typename InputType = arma::mat, -// typename OutputType = arma::mat -// > -// using SigmoidLayer = BaseLayer; typedef BaseLayer Sigmoid; -typedef BaseLayer SigmoidLayer; +template< + typename InputType = arma::mat, + typename OutputType = arma::mat +> +using SigmoidLayer = BaseLayer; /** * Standard Identity-Layer using the identity activation function. */ -// template < -// class ActivationFunction = IdentityFunction, -// typename InputDataType = arma::mat, -// typename OutputDataType = arma::mat -// > -// using IdentityLayer = BaseLayer< -// ActivationFunction, InputDataType, OutputDataType>; -typedef BaseLayer IdentityLayer; typedef BaseLayer Identity; +template < + typename InputType = arma::mat, + typename OutputType = arma::mat +> +using IdentityLayer = BaseLayer; /** * Standard rectified linear unit non-linearity layer. */ -// template < -// class ActivationFunction = RectifierFunction, -// typename InputDataType = arma::mat, -// typename OutputDataType = arma::mat -// > -// using ReLULayer = BaseLayer< -// ActivationFunction, InputDataType, OutputDataType>; -typedef BaseLayer ReLULayer; typedef BaseLayer ReLU; +template< + typename InputType = arma::mat, + typename OutputType = arma::mat +> +using ReLULayer = BaseLayer; /** * Standard hyperbolic tangent layer. */ -// template < -// class ActivationFunction = TanhFunction, -// typename InputDataType = arma::mat, -// typename OutputDataType = arma::mat -// > -// using TanHLayer = BaseLayer< -// ActivationFunction, InputDataType, OutputDataType>; -typedef BaseLayer TanHLayer; +template< + typename InputType = arma::mat, + typename OutputType = arma::mat +> +using TanHLayer = BaseLayer; /** * Standard Softplus-Layer using the Softplus activation function. */ -// template < -// class ActivationFunction = SoftplusFunction, -// typename InputDataType = arma::mat, -// typename OutputDataType = arma::mat -// > -// using SoftPlusLayer = BaseLayer< -// ActivationFunction, InputDataType, OutputDataType>; -typedef BaseLayer SoftPlusLayer; +template< + typename InputType = arma::mat, + typename OutputType = arma::mat +> +using SoftPlusLayer = BaseLayer; /** * Standard HardSigmoid-Layer using the HardSigmoid activation function. */ -// template < -// class ActivationFunction = HardSigmoidFunction, -// typename InputDataType = arma::mat, -// typename OutputDataType = arma::mat -// > -// using HardSigmoidLayer = BaseLayer< -// ActivationFunction, InputDataType, OutputDataType>; -typedef BaseLayer HardSigmoidLayer; +template< + typename InputType = arma::mat, + typename OutputType = arma::mat +> +using HardSigmoidLayer = BaseLayer; /** * Standard Swish-Layer using the Swish activation function. */ -// template < -// class ActivationFunction = SwishFunction, -// typename InputDataType = arma::mat, -// typename OutputDataType = arma::mat -// > -// using SwishFunctionLayer = BaseLayer< -// ActivationFunction, InputDataType, OutputDataType>; -typedef BaseLayer SwishFunctionLayer; +template< + typename InputType = arma::mat, + typename OutputType = arma::mat +> +using SwishFunctionLayer = BaseLayer; /** * Standard Mish-Layer using the Mish activation function. */ -// template < -// class ActivationFunction = MishFunction, -// typename InputDataType = arma::mat, -// typename OutputDataType = arma::mat -// > -// using MishFunctionLayer = BaseLayer< -// ActivationFunction, InputDataType, OutputDataType>; -typedef BaseLayer MishFunctionLayer; +template< + typename InputType = arma::mat, + typename OutputType = arma::mat +> +using MishFunctionLayer = BaseLayer; /** * Standard LiSHT-Layer using the LiSHT activation function. */ -// template < -// class ActivationFunction = LiSHTFunction, -// typename InputDataType = arma::mat, -// typename OutputDataType = arma::mat -// > -// using LiSHTFunctionLayer = BaseLayer< -// ActivationFunction, InputDataType, OutputDataType>; -typedef BaseLayer LiSHTFunctionLayer; +template< + typename InputType = arma::mat, + typename OutputType = arma::mat +> +using LiSHTFunctionLayer = BaseLayer; /** * Standard GELU-Layer using the GELU activation function. */ -// template < -// class ActivationFunction = GELUFunction, -// typename InputDataType = arma::mat, -// typename OutputDataType = arma::mat -// > -// using GELUFunctionLayer = BaseLayer< -// ActivationFunction, InputDataType, OutputDataType>; -typedef BaseLayer GELUFunctionLayer; +template < + typename InputType = arma::mat, + typename OutputType = arma::mat +> +using GELUFunctionLayer = BaseLayer; /** * Standard Elliot-Layer using the Elliot activation function. */ -// template < -// class ActivationFunction = ElliotFunction, -// typename InputDataType = arma::mat, -// typename OutputDataType = arma::mat -// > -// using ElliotFunctionLayer = BaseLayer< -// ActivationFunction, InputDataType, OutputDataType>; -typedef BaseLayer ElliotFunctionLayer; +template < + typename InputType = arma::mat, + typename OutputType = arma::mat +> +using ElliotFunctionLayer = BaseLayer; /** * Standard ELiSH-Layer using the ELiSH activation function. */ -// template < -// class ActivationFunction = ElishFunction, -// typename InputDataType = arma::mat, -// typename OutputDataType = arma::mat -// > -// using ElishFunctionLayer = BaseLayer< -// ActivationFunction, InputDataType, OutputDataType>; -typedef BaseLayer ElishFunctionLayer; +template < + typename InputType = arma::mat, + typename OutputType = arma::mat +> +using ElishFunctionLayer = BaseLayer; /** * Standard Gaussian-Layer using the Gaussian activation function. */ -// template < -// class ActivationFunction = GaussianFunction, -// typename InputDataType = arma::mat, -// typename OutputDataType = arma::mat -// > -// using GaussianFunctionLayer = BaseLayer< -// ActivationFunction, InputDataType, OutputDataType>; -typedef BaseLayer GaussianFunctionLayer; +template < + typename InputType = arma::mat, + typename OutputType = arma::mat +> +using GaussianFunctionLayer = BaseLayer; } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/batch_norm.hpp b/src/mlpack/methods/ann/layer/batch_norm.hpp index 06989ed5f1..24b19f06e3 100644 --- a/src/mlpack/methods/ann/layer/batch_norm.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm.hpp @@ -131,7 +131,7 @@ class BatchNorm : public Layer OutputType& Gradient() { return gradient; } //! Get the value of deterministic parameter. - bool Deterministic() const { return deterministic; } + const bool& Deterministic() const { return deterministic; } //! Modify the value of deterministic parameter. bool& Deterministic() { return deterministic; } diff --git a/src/mlpack/methods/ann/layer/concat_performance.hpp b/src/mlpack/methods/ann/layer/concat_performance.hpp index 8fa0c4ff07..72f8c80878 100644 --- a/src/mlpack/methods/ann/layer/concat_performance.hpp +++ b/src/mlpack/methods/ann/layer/concat_performance.hpp @@ -54,7 +54,7 @@ class ConcatPerformance : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - double Forward(const InputType& input, OutputType& target); + void Forward(const InputType& input, OutputType& target); /** * Ordinary feed backward pass of a neural network. The negative log diff --git a/src/mlpack/methods/ann/layer/concat_performance_impl.hpp b/src/mlpack/methods/ann/layer/concat_performance_impl.hpp index b62ab5857a..355b8afe02 100644 --- a/src/mlpack/methods/ann/layer/concat_performance_impl.hpp +++ b/src/mlpack/methods/ann/layer/concat_performance_impl.hpp @@ -39,23 +39,24 @@ template< typename InputType, typename OutputType > -template -double ConcatPerformance< +void ConcatPerformance< OutputLayerType, InputType, OutputType ->::Forward(const arma::Mat& input, arma::Mat& target) +>::Forward(const InputType& input, OutputType& target) { const size_t elements = input.n_elem / inSize; double output = 0; - for (size_t i = 0; i < input.n_elem; i+= elements) + for (size_t i = 0; i < input.n_elem; i += elements) { - arma::mat subInput = input.submat(i, 0, i + elements - 1, 0); + InputType subInput = input.submat(i, 0, i + elements - 1, 0); output += outputLayer.Forward(subInput, target); } - return output; + // TODO: what to do with output? + //return output; + return; } template< @@ -63,20 +64,19 @@ template< typename InputType, typename OutputType > -template void ConcatPerformance< OutputLayerType, InputType, OutputType >::Backward( - const arma::Mat& input, - const arma::Mat& target, - arma::Mat& output) + const InputType& input, + const OutputType& target, + OutputType& output) { const size_t elements = input.n_elem / inSize; - arma::mat subInput = input.submat(0, 0, elements - 1, 0); - arma::mat subOutput; + InputType subInput = input.submat(0, 0, elements - 1, 0); + OutputType subOutput; outputLayer.Backward(subInput, target, subOutput); diff --git a/src/mlpack/methods/ann/layer/concatenate.hpp b/src/mlpack/methods/ann/layer/concatenate.hpp index 56d3383861..5405ad60ed 100644 --- a/src/mlpack/methods/ann/layer/concatenate.hpp +++ b/src/mlpack/methods/ann/layer/concatenate.hpp @@ -102,7 +102,7 @@ class ConcatenateType : public Layer * Serialize the layer. */ template - void serialize(Archive& /* ar */, const uint32_t /* version */) + void serialize(Archive& ar, const uint32_t /* version */) { ar(cereal::base_class>(this)); } diff --git a/src/mlpack/methods/ann/layer/constant.hpp b/src/mlpack/methods/ann/layer/constant.hpp index cb1264b12d..cb951142ce 100644 --- a/src/mlpack/methods/ann/layer/constant.hpp +++ b/src/mlpack/methods/ann/layer/constant.hpp @@ -34,6 +34,11 @@ template class ConstantType : public Layer { public: + /** + * Create an empty Constant layer. + */ + ConstantType(); + /** * Create the Constant object that outputs a given constant scalar value * given any input value. @@ -43,8 +48,18 @@ class ConstantType : public Layer */ ConstantType(const size_t outSize, const double scalar = 0); + //! Copy another ConstantType. + ConstantType(const ConstantType& layer); + //! Take ownership of another ConstantType. + ConstantType(ConstantType&& layer); + //! Copy another ConstantType. + ConstantType& operator=(const ConstantType& layer); + //! Take ownership of another ConstantType. + ConstantType& operator=(ConstantType&& layer); + //! Clone the ConstantType object. This handles polymorphism correctly. ConstantType* Clone() const { return new ConstantType(*this); } + /** * Ordinary feed forward pass of a neural network. The forward pass fills the * output with the specified constant parameter. diff --git a/src/mlpack/methods/ann/layer/constant_impl.hpp b/src/mlpack/methods/ann/layer/constant_impl.hpp index a712721a48..2af1b2da59 100644 --- a/src/mlpack/methods/ann/layer/constant_impl.hpp +++ b/src/mlpack/methods/ann/layer/constant_impl.hpp @@ -19,6 +19,14 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { +template +ConstantType::ConstantType() : + inSize(0), + outSize(0) +{ + // Nothing to do. +} + template ConstantType::ConstantType( const size_t outSize, @@ -30,6 +38,62 @@ ConstantType::ConstantType( constantOutput.fill(scalar); } +template +ConstantType::ConstantType( + const ConstantType& other) : + inSize(other.inSize), + outSize(other.outSize), + constantOutput(other.constantOutput) +{ + // Nothing else to do. +} + +template +ConstantType::ConstantType( + ConstantType&& other) : + inSize(other.inSize), + outSize(other.outSize), + constantOutput(std::move(other.constantOutput)) +{ + other.inSize = 0; + other.outSize = 1; + other.constantOutput = OutputType(other.outSize, 1); +} + +template +ConstantType& +ConstantType::operator=( + const ConstantType& other) +{ + if (this != &other) + { + inSize = other.inSize; + outSize = other.outSize; + constantOutput = other.constantOutput; + } + + return *this; +} + +template +ConstantType& +ConstantType::operator=( + ConstantType&& other) +{ + if (this != *other) + { + inSize = other.inSize; + outSize = other.outSize; + constantOutput = std::move(other.constantOutput); + + other.inSize = 0; + other.outSize = 1; + other.constantOutput = OutputType(other.outSize, 1); + } + + return *this; +} + template void ConstantType::Forward( const InputType& input, OutputType& output) diff --git a/src/mlpack/methods/ann/layer/dropout.hpp b/src/mlpack/methods/ann/layer/dropout.hpp index 7224611693..316066512d 100644 --- a/src/mlpack/methods/ann/layer/dropout.hpp +++ b/src/mlpack/methods/ann/layer/dropout.hpp @@ -138,8 +138,6 @@ typedef DropoutType Dropout; } // namespace ann } // namespace mlpack -CEREAL_REGISTER_TYPE(mlpack::ann::Dropout); - // Include implementation. #include "dropout_impl.hpp" diff --git a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp index 33dedb40f6..09ea7a5085 100644 --- a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp @@ -134,14 +134,14 @@ void FastLSTMType::Reset() { // Set the weight parameter for the input to gate layer (linear layer) using // the overall layer parameter matrix. - input2GateWeight = OutputDataType(weights.memptr(), + input2GateWeight = OutputType(weights.memptr(), 4 * outSize, inSize, false, false); - input2GateBias = OutputDataType(weights.memptr() + input2GateWeight.n_elem, + input2GateBias = OutputType(weights.memptr() + input2GateWeight.n_elem, 4 * outSize, 1, false, false); // Set the weight parameter for the output to gate layer // (linear no bias layer) using the overall layer parameter matrix. - output2GateWeight = OutputDataType(weights.memptr() + input2GateWeight.n_elem + output2GateWeight = OutputType(weights.memptr() + input2GateWeight.n_elem + input2GateBias.n_elem, 4 * outSize, outSize, false, false); } @@ -210,8 +210,8 @@ void FastLSTMType::Forward( forwardStep, forwardStep + batchStep); gate.cols(forwardStep, forwardStep + batchStep).each_col() += input2GateBias; - arma::subview sigmoidOut = gateActivation.cols(forwardStep, - forwardStep + batchStep); + InputType sigmoidOut(gateActivation.colptr(forwardStep), + gateActivation.n_rows, batchStep, false, false); FastSigmoid( gate.submat(0, forwardStep, 3 * outSize - 1, forwardStep + batchStep), sigmoidOut); diff --git a/src/mlpack/methods/ann/layer/gru.hpp b/src/mlpack/methods/ann/layer/gru.hpp index dfb3357b5c..23f6da22fb 100644 --- a/src/mlpack/methods/ann/layer/gru.hpp +++ b/src/mlpack/methods/ann/layer/gru.hpp @@ -31,9 +31,6 @@ #include -#include "../visitor/delta_visitor.hpp" -#include "../visitor/output_parameter_visitor.hpp" - #include "layer_types.hpp" #include "add_merge.hpp" #include "sequential.hpp" @@ -101,10 +98,9 @@ class GRU : public Layer * @param error The calculated error. * @param gradient The calculated gradient. */ - template - void Gradient(const arma::Mat& input, - const arma::Mat& /* error */, - arma::Mat& /* gradient */); + void Gradient(const InputType& input, + const OutputType& /* error */, + OutputType& /* gradient */); /* * Resets the cell to accept a new input. This breaks the BPTT chain starts a @@ -115,7 +111,7 @@ class GRU : public Layer void ResetCell(const size_t size); //! The value of the deterministic parameter. - bool Deterministic() const { return deterministic; } + const bool& Deterministic() const { return deterministic; } //! Modify the value of the deterministic parameter. bool& Deterministic() { return deterministic; } @@ -212,13 +208,13 @@ class GRU : public Layer OutputType allZeros; //! Iterator pointed to the last output produced by the cell - std::list::iterator prevOutput; + typename std::list::iterator prevOutput; //! Iterator pointed to the last output processed by backward - std::list::iterator backIterator; + typename std::list::iterator backIterator; //! Iterator pointed to the last output processed by gradient - std::list::iterator gradIterator; + typename std::list::iterator gradIterator; //! Locally-stored previous error. OutputType prevError; diff --git a/src/mlpack/methods/ann/layer/gru_impl.hpp b/src/mlpack/methods/ann/layer/gru_impl.hpp index 0f3856daf0..b738d0ff99 100644 --- a/src/mlpack/methods/ann/layer/gru_impl.hpp +++ b/src/mlpack/methods/ann/layer/gru_impl.hpp @@ -16,10 +16,6 @@ // In case it hasn't yet been included. #include "gru.hpp" -#include "../visitor/forward_visitor.hpp" -#include "../visitor/backward_visitor.hpp" -#include "../visitor/gradient_visitor.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -44,21 +40,23 @@ GRU::GRU( deterministic(false) { // Input specific linear layers(for zt, rt, ot). - input2GateModule = new Linear<>(inSize, 3 * outSize); + input2GateModule = new LinearType(inSize, 3 * outSize); // Previous output gates (for zt and rt). - output2GateModule = new LinearNoBias<>(outSize, 2 * outSize); + output2GateModule = new LinearNoBiasType(outSize, + 2 * outSize); // Previous output gate for ot. - outputHidden2GateModule = new LinearNoBias<>(outSize, outSize); + outputHidden2GateModule = new LinearNoBiasType(outSize, + outSize); network.push_back(input2GateModule); network.push_back(output2GateModule); network.push_back(outputHidden2GateModule); - inputGateModule = new SigmoidLayer<>(); - forgetGateModule = new SigmoidLayer<>(); - hiddenStateModule = new TanHLayer<>(); + inputGateModule = new SigmoidLayer(); + forgetGateModule = new SigmoidLayer(); + hiddenStateModule = new TanHLayer(); network.push_back(inputGateModule); network.push_back(hiddenStateModule); @@ -139,7 +137,7 @@ void GRU::Forward( // cmul2 is (1 - input gate) * hidden gate. output = (inputGateModule->OutputParameter() % (*prevOutput - hiddenStateModule->OutputParameter())) + - % hiddenStateModule->OutputParameter(); + hiddenStateModule->OutputParameter(); forwardStep++; if (forwardStep == rho) @@ -225,7 +223,7 @@ void GRU::Backward( // Delta ot. OutputType dOt = gyLocal % (arma::ones(outSize, batchSize) - - inputGateModule->OutputParameter(); + inputGateModule->OutputParameter()); // Delta of input gate. inputGateModule->Backward(inputGateModule->OutputParameter(), dZt, diff --git a/src/mlpack/methods/ann/layer/highway_impl.hpp b/src/mlpack/methods/ann/layer/highway_impl.hpp index f57903fac7..58f2a4014c 100644 --- a/src/mlpack/methods/ann/layer/highway_impl.hpp +++ b/src/mlpack/methods/ann/layer/highway_impl.hpp @@ -208,6 +208,14 @@ void HighwayType::serialize( ar(CEREAL_NVP(weights)); ar(CEREAL_NVP(model)); ar(CEREAL_VECTOR_POINTER(network)); + + // Reset the memory. + if (Archive::is_loading::value) + { + networkOwnerships.clear(); + networkOwnerships.resize(network.size(), true); + Reset(); + } } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/layer.hpp b/src/mlpack/methods/ann/layer/layer.hpp index e53c94f1e8..abe123021b 100644 --- a/src/mlpack/methods/ann/layer/layer.hpp +++ b/src/mlpack/methods/ann/layer/layer.hpp @@ -12,6 +12,9 @@ #ifndef MLPACK_METHODS_ANN_LAYER_LAYER_HPP #define MLPACK_METHODS_ANN_LAYER_LAYER_HPP +namespace mlpack { +namespace ann { + /** * A layer is an abstract class implementing common neural networks operations, * such as convolution, batch norm, etc. These operations require managing @@ -292,4 +295,7 @@ class Layer }; +} // namespace ann +} // namespace mlpack + #endif diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 18907ba25a..9310a1e872 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -12,6 +12,8 @@ #ifndef MLPACK_METHODS_ANN_LAYER_LAYER_TYPES_HPP #define MLPACK_METHODS_ANN_LAYER_LAYER_TYPES_HPP +#include + // Layer modules. #include #include @@ -44,6 +46,8 @@ #include #include #include +#include +#include #include #include #include @@ -53,12 +57,15 @@ #include #include #include -// #include +#include +#include +#include #include #include #include #include #include +#include #include #include #include @@ -66,16 +73,24 @@ #include #include -// // Convolution modules. -// #include -// #include -// #include +// Depends on PaddingType<>. +#include +// Depends on Linear<> and LinearNoBias<>. +#include -// // Regularizers. -// #include +// Convolution modules. +#include +#include +#include -// // Loss function modules. -// #include +// Regularizers. +#include + +// Loss function modules. +#include + +// Depends on NegativeLogLikelihood. +#include // Include definitions for polymorphic serialization. #include diff --git a/src/mlpack/methods/ann/layer/linear.hpp b/src/mlpack/methods/ann/layer/linear.hpp index 4868bdacc4..70a3e3b152 100644 --- a/src/mlpack/methods/ann/layer/linear.hpp +++ b/src/mlpack/methods/ann/layer/linear.hpp @@ -179,8 +179,6 @@ typedef LinearType Linear; } // namespace ann } // namespace mlpack -CEREAL_REGISTER_TYPE(mlpack::ann::Linear); - // Include implementation. #include "linear_impl.hpp" diff --git a/src/mlpack/methods/ann/layer/log_softmax.hpp b/src/mlpack/methods/ann/layer/log_softmax.hpp index f1914a9505..331b3dd70a 100644 --- a/src/mlpack/methods/ann/layer/log_softmax.hpp +++ b/src/mlpack/methods/ann/layer/log_softmax.hpp @@ -84,8 +84,6 @@ typedef LogSoftMaxType LogSoftMax; } // namespace ann } // namespace mlpack -CEREAL_REGISTER_TYPE(mlpack::ann::LogSoftMax); - // Include implementation. #include "log_softmax_impl.hpp" diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index 2cc1202cbe..cf1b1d9bdc 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -59,7 +59,7 @@ template < typename InputType = arma::mat, typename OutputType = arma::mat > -class LSTM +class LSTM : public Layer { public: //! Create the LSTM object. @@ -95,7 +95,6 @@ class LSTM * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - template void Forward(const InputType& input, OutputType& output); /** @@ -107,7 +106,6 @@ class LSTM * @param cellState Cell state of the LSTM. * @param useCellState Use the cellState passed in the LSTM cell. */ - template void Forward(const InputType& input, OutputType& output, OutputType& cellState, @@ -122,10 +120,9 @@ class LSTM * @param gy The backpropagated error. * @param g The calculated gradient. */ - template void Backward(const InputType& input, - const ErrorType& gy, - GradientType& g); + const OutputType& gy, + OutputType& g); /* * Reset the layer parameter. @@ -147,10 +144,9 @@ class LSTM * @param error The calculated error. * @param gradient The calculated gradient. */ - template void Gradient(const InputType& input, - const ErrorType& error, - GradientType& gradient); + const OutputType& error, + OutputType& gradient); //! Get the maximum number of steps to backpropagate through time (BPTT). size_t Rho() const { return rho; } diff --git a/src/mlpack/methods/ann/layer/lstm_impl.hpp b/src/mlpack/methods/ann/layer/lstm_impl.hpp index ab40fb3ce3..89c56b5893 100644 --- a/src/mlpack/methods/ann/layer/lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/lstm_impl.hpp @@ -26,7 +26,7 @@ LSTM::LSTM() template LSTM::LSTM( - const LSTM& layer) : + const LSTM& layer) : inSize(layer.inSize), outSize(layer.outSize), rho(layer.rho), @@ -45,7 +45,7 @@ LSTM::LSTM( template LSTM::LSTM( - LSTM&& layer) : + LSTM&& layer) : inSize(std::move(layer.inSize)), outSize(std::move(layer.outSize)), rho(std::move(layer.rho)), @@ -63,8 +63,8 @@ LSTM::LSTM( } template -LSTM& -LSTM :: operator=(const LSTM& layer) +LSTM& +LSTM::operator=(const LSTM& layer) { if (this != &layer) { @@ -82,12 +82,12 @@ LSTM :: operator=(const LSTM& layer) rhoSize = layer.rho; bpttSteps = layer.bpttSteps; } - return *this; + return *this; } template -LSTM& -LSTM :: operator=(LSTM&& layer) +LSTM& +LSTM::operator=(LSTM&& layer) { if (this != &layer) { @@ -105,7 +105,7 @@ LSTM :: operator=(LSTM&& layer) rhoSize = std::move(layer.rho); bpttSteps = std::move(layer.bpttSteps); } - return *this; + return *this; } template @@ -245,7 +245,6 @@ void LSTM::Reset() // Forward when cellState is not needed. template -template void LSTM::Forward( const InputType& input, OutputType& output) { @@ -256,11 +255,10 @@ void LSTM::Forward( // Forward when cellState is needed overloaded LSTM::Forward(). template -template void LSTM::Forward(const InputType& input, - OutputType& output, - OutputType& cellState, - bool useCellState) + OutputType& output, + OutputType& cellState, + bool useCellState) { // Check if the batch size changed, the number of cols is defines the input // batch size. @@ -370,11 +368,10 @@ void LSTM::Forward(const InputType& input, } template -template void LSTM::Backward( - const InputType& /* input */, const ErrorType& gy, GradientType& g) + const InputType& /* input */, const OutputType& gy, OutputType& g) { - ErrorType gyLocal; + OutputType gyLocal; if (gradientStepIdx > 0) { gyLocal = gy + prevError; @@ -382,8 +379,8 @@ void LSTM::Backward( else { // Make an alias. - gyLocal = ErrorType(((ErrorType&) gy).memptr(), gy.n_rows, gy.n_cols, false, - false); + gyLocal = OutputType(((OutputType&) gy).memptr(), gy.n_rows, gy.n_cols, + false, false); } outputGateError = @@ -448,11 +445,10 @@ void LSTM::Backward( } template -template void LSTM::Gradient( const InputType& input, - const ErrorType& /* error */, - GradientType& gradient) + const OutputType& /* error */, + OutputType& gradient) { // Input2GateOutputWeight and input2GateOutputBias gradients. gradient.submat(0, 0, input2GateOutputWeight.n_elem - 1, 0) = diff --git a/src/mlpack/methods/ann/layer/max_pooling.hpp b/src/mlpack/methods/ann/layer/max_pooling.hpp index 751deb75ac..dea861bd5d 100644 --- a/src/mlpack/methods/ann/layer/max_pooling.hpp +++ b/src/mlpack/methods/ann/layer/max_pooling.hpp @@ -70,6 +70,10 @@ class MaxPoolingType : public Layer const size_t strideHeight = 1, const bool floor = true); + // TODO: copy constructor / move constructor + + MaxPoolingType* Clone() const { return new MaxPoolingType(*this); } + /** * Ordinary feed forward pass of a neural network, evaluating the function * f(x) by propagating the activity forward through f. diff --git a/src/mlpack/methods/ann/layer/mean_pooling.hpp b/src/mlpack/methods/ann/layer/mean_pooling.hpp index 2a10e5f860..24772499c9 100644 --- a/src/mlpack/methods/ann/layer/mean_pooling.hpp +++ b/src/mlpack/methods/ann/layer/mean_pooling.hpp @@ -51,6 +51,9 @@ class MeanPoolingType : public Layer const size_t strideHeight = 1, const bool floor = true); + // TODO: copy constructor / move constructor + MeanPoolingType* Clone() const { return new MeanPoolingType(*this); } + /** * Ordinary feed forward pass of a neural network, evaluating the function * f(x) by propagating the activity forward through f. diff --git a/src/mlpack/methods/ann/layer/padding.hpp b/src/mlpack/methods/ann/layer/padding.hpp index 91e2879536..1ce0ef358b 100644 --- a/src/mlpack/methods/ann/layer/padding.hpp +++ b/src/mlpack/methods/ann/layer/padding.hpp @@ -47,8 +47,8 @@ class PaddingType : public Layer const size_t padHTop = 0, const size_t padHBottom = 0); - //! Clone the PaddingType object. This handles polymorphism correctly. - PaddingType* Clone() const { return new PaddingType(*this); } + //! Clone the PaddingType object. This handles polymorphism correctly. + PaddingType* Clone() const { return new PaddingType(*this); } /** * Ordinary feed forward pass of a neural network, evaluating the function diff --git a/src/mlpack/methods/ann/layer/recurrent.hpp b/src/mlpack/methods/ann/layer/recurrent.hpp index 8191fdccea..7a57b7ce4a 100644 --- a/src/mlpack/methods/ann/layer/recurrent.hpp +++ b/src/mlpack/methods/ann/layer/recurrent.hpp @@ -14,11 +14,6 @@ #include -#include "../visitor/delete_visitor.hpp" -#include "../visitor/delta_visitor.hpp" -#include "../visitor/copy_visitor.hpp" -#include "../visitor/output_parameter_visitor.hpp" - #include "layer_types.hpp" #include "add_merge.hpp" #include "sequential.hpp" @@ -37,7 +32,7 @@ namespace ann /** Artificial Neural Network. */ { */ template < typename InputType = arma::mat, - typename OutputType = arma::mat, + typename OutputType = arma::mat > class Recurrent : public Layer { @@ -107,7 +102,7 @@ class Recurrent : public Layer std::vector*>& Model() { return network; } //! The value of the deterministic parameter. - bool Deterministic() const { return deterministic; } + const bool& Deterministic() const { return deterministic; } //! Modify the value of the deterministic parameter. bool& Deterministic() { return deterministic; } @@ -141,9 +136,6 @@ class Recurrent : public Layer void serialize(Archive& ar, const uint32_t /* version */); private: - //! Locally-stored delete visitor module object. - DeleteVisitor deleteVisitor; - //! Locally-stored start module. Layer* startModule; @@ -179,10 +171,10 @@ class Recurrent : public Layer OutputType parameters; //! Locally-stored initial module. - Sequential* initialModule; + SequentialType* initialModule; //! Locally-stored recurrent module. - Sequential* recurrentModule; + SequentialType* recurrentModule; //! Locally-stored model modules. std::vector*> network; diff --git a/src/mlpack/methods/ann/layer/recurrent_attention.hpp b/src/mlpack/methods/ann/layer/recurrent_attention.hpp index b493192066..337cadc5eb 100644 --- a/src/mlpack/methods/ann/layer/recurrent_attention.hpp +++ b/src/mlpack/methods/ann/layer/recurrent_attention.hpp @@ -13,12 +13,6 @@ #define MLPACK_METHODS_ANN_LAYER_RECURRENT_ATTENTION_HPP #include -#include - -#include "../visitor/delta_visitor.hpp" -#include "../visitor/output_parameter_visitor.hpp" -#include "../visitor/reset_visitor.hpp" -#include "../visitor/weight_size_visitor.hpp" #include "layer_types.hpp" #include "add_merge.hpp" @@ -110,10 +104,10 @@ class RecurrentAttention : public Layer OutputType& /* gradient */); //! Get the model modules. - std::vector>& Model() { return network; } + std::vector*>& Model() { return network; } //! The value of the deterministic parameter. - bool Deterministic() const { return deterministic; } + const bool& Deterministic() const { return deterministic; } //! Modify the value of the deterministic parameter. bool& Deterministic() { return deterministic; } diff --git a/src/mlpack/methods/ann/layer/recurrent_attention_impl.hpp b/src/mlpack/methods/ann/layer/recurrent_attention_impl.hpp index ff003b2e60..8d401d0767 100644 --- a/src/mlpack/methods/ann/layer/recurrent_attention_impl.hpp +++ b/src/mlpack/methods/ann/layer/recurrent_attention_impl.hpp @@ -15,14 +15,6 @@ // In case it hasn't yet been included. #include "recurrent_attention.hpp" -#include "../visitor/load_output_parameter_visitor.hpp" -#include "../visitor/save_output_parameter_visitor.hpp" -#include "../visitor/backward_visitor.hpp" -#include "../visitor/forward_visitor.hpp" -#include "../visitor/gradient_set_visitor.hpp" -#include "../visitor/gradient_update_visitor.hpp" -#include "../visitor/gradient_visitor.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -97,7 +89,7 @@ void RecurrentAttention::Forward( } } - output = boost::apply_visitor(outputParameterVisitor, rnnModule); + output = rnnModule->OutputParameter(); forwardStep = 0; backwardStep = 0; diff --git a/src/mlpack/methods/ann/layer/recurrent_impl.hpp b/src/mlpack/methods/ann/layer/recurrent_impl.hpp index e801dc1a9e..891a0473de 100644 --- a/src/mlpack/methods/ann/layer/recurrent_impl.hpp +++ b/src/mlpack/methods/ann/layer/recurrent_impl.hpp @@ -15,11 +15,6 @@ // In case it hasn't yet been included. #include "recurrent.hpp" -#include "../visitor/add_visitor.hpp" -#include "../visitor/backward_visitor.hpp" -#include "../visitor/gradient_visitor.hpp" -#include "../visitor/gradient_zero_visitor.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -60,9 +55,9 @@ Recurrent::Recurrent( deterministic(false), ownsLayer(true) { - initialModule = new Sequential<>(); - mergeModule = new AddMerge<>(false, false, false); - recurrentModule = new Sequential<>(false, false); + initialModule = new SequentialType(); + mergeModule = new AddMerge(false, false, false); + recurrentModule = new SequentialType(false, false); initialModule->Add(inputModule); initialModule->Add(startModule); @@ -95,9 +90,9 @@ Recurrent::Recurrent( feedbackModule = network.feedbackModule->Clone(); transferModule = network.transferModule->Clone(); - initialModule = new Sequential<>(); - mergeModule = new AddMerge<>(false, false, false); - recurrentModule = new Sequential<>(false, false); + initialModule = new SequentialType(); + mergeModule = new AddMerge(false, false, false); + recurrentModule = new SequentialType(false, false); initialModule->Add(inputModule); initialModule->Add(startModule); @@ -116,7 +111,7 @@ Recurrent::Recurrent( } template -void Recurrent::Forward( +void Recurrent::Forward( const InputType& input, OutputType& output) { if (forwardStep == 0) @@ -152,9 +147,8 @@ void Recurrent::Forward( } } -template -void Recurrent::Backward( +template +void Recurrent::Backward( const InputType& /* input */, const OutputType& gy, OutputType& g) { if (!recurrentError.is_empty()) @@ -185,9 +179,8 @@ void Recurrent::Backward( backwardStep++; } -template -void Recurrent::Gradient( +template +void Recurrent::Gradient( const InputType& input, const OutputType& error, OutputType& /* gradient */) @@ -219,10 +212,9 @@ void Recurrent::Gradient( } } -template +template template -void Recurrent::serialize( +void Recurrent::serialize( Archive& ar, const uint32_t /* version */) { ar(cereal::base_class>(this)); @@ -246,9 +238,9 @@ void Recurrent::serialize( // Set up the network. if (cereal::is_loading()) { - initialModule = new Sequential<>(); - mergeModule = new AddMerge<>(false, false, false); - recurrentModule = new Sequential<>(false, false); + initialModule = new SequentialType(); + mergeModule = new AddMerge(false, false, false); + recurrentModule = new SequentialType(false, false); initialModule->Add(inputModule); initialModule->Add(startModule); diff --git a/src/mlpack/methods/ann/layer/serialization.hpp b/src/mlpack/methods/ann/layer/serialization.hpp index 414a521fe4..cee9dfa4cc 100644 --- a/src/mlpack/methods/ann/layer/serialization.hpp +++ b/src/mlpack/methods/ann/layer/serialization.hpp @@ -21,37 +21,19 @@ mlpack::ann::NaiveConvolution, \ mlpack::ann::NaiveConvolution, \ __VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer); \ - CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer); \ - CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer< \ - mlpack::ann::RectifierFunction, \ - __VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer); \ - CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer); \ - CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer< \ - mlpack::ann::HardSigmoidFunction, \ - __VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer< \ - mlpack::ann::RectifierFunction, \ - __VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer); \ - CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer); \ - CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer); \ - CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer); \ - CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer); \ - CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer); \ - CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer); \ + CEREAL_REGISTER_TYPE(mlpack::ann::SigmoidLayer<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::ann::IdentityLayer<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::ann::ReLULayer<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::ann::TanHLayer<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::ann::SoftPlusLayer<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::ann::HardSigmoidLayer<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::ann::SwishFunctionLayer<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::ann::MishFunctionLayer<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::ann::LiSHTFunctionLayer<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::ann::GELUFunctionLayer<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::ann::ElliotFunctionLayer<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::ann::ElishFunctionLayer<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::ann::GaussianFunctionLayer<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::BatchNorm<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::BilinearInterpolationType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::CELUType<__VA_ARGS__>); \ @@ -97,7 +79,7 @@ CEREAL_REGISTER_TYPE(mlpack::ann::PositionalEncodingType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::RBF<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::RecurrentAttention<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::RecurrentType<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::ann::Recurrent<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::ReinforceNormalType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::ReparametrizationType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::SelectType<__VA_ARGS__>); \ @@ -117,6 +99,8 @@ // TODO: continue... +CEREAL_REGISTER_MLPACK_LAYERS(arma::mat, arma::mat); + // TODO: I think this below is not needed. /** * Register an mlpack layer with the given INPUT_TYPE and OUTPUT_TYPE. diff --git a/src/mlpack/methods/ann/layer/spatial_dropout.hpp b/src/mlpack/methods/ann/layer/spatial_dropout.hpp index ca359d0dfd..6b8b9dc6c0 100644 --- a/src/mlpack/methods/ann/layer/spatial_dropout.hpp +++ b/src/mlpack/methods/ann/layer/spatial_dropout.hpp @@ -86,7 +86,7 @@ class SpatialDropoutType : public Layer size_t& Size() { return size; } //! Get the value of the deterministic parameter. - bool Deterministic() const { return deterministic; } + const bool& Deterministic() const { return deterministic; } //! Modify the value of the deterministic parameter. bool& Deterministic() { return deterministic; } diff --git a/src/mlpack/methods/ann/layer/weight_norm.hpp b/src/mlpack/methods/ann/layer/weight_norm.hpp index ad0ff58cfb..1bd76d3363 100644 --- a/src/mlpack/methods/ann/layer/weight_norm.hpp +++ b/src/mlpack/methods/ann/layer/weight_norm.hpp @@ -52,6 +52,11 @@ template < class WeightNormType : public Layer { public: + /** + * Create an empty WeightNorm layer. + */ + WeightNormType(); + /** * Create the WeightNorm layer object. * @@ -62,6 +67,15 @@ class WeightNormType : public Layer //! Destructor to release allocated memory. ~WeightNormType(); + //! Create a WeightNorm layer by copying the given layer. + WeightNormType(const WeightNormType& other); + //! Create a WeightNorm layer by taking ownership of the other layer. + WeightNormType(WeightNormType&& other); + //! Copy the given layer. + WeightNormType& operator=(const WeightNormType& other); + //! Take ownership of the data in the given layer. + WeightNormType& operator=(WeightNormType&& other); + //! Clone the WeightNormType object. This handles polymorphism correctly. WeightNormType* Clone() const { return new WeightNormType(*this); } diff --git a/src/mlpack/methods/ann/layer/weight_norm_impl.hpp b/src/mlpack/methods/ann/layer/weight_norm_impl.hpp index 5c89904c83..30012e66c0 100644 --- a/src/mlpack/methods/ann/layer/weight_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/weight_norm_impl.hpp @@ -19,6 +19,17 @@ namespace mlpack { namespace ann { /** Artificial Neural Network. */ +template +WeightNormType:: +WeightNormType() : wrappedLayer(new LinearType()) +{ + layerWeightSize = wrappedLayer->WeightSize(); + weights.set_size(layerWeightSize + 1, 1); + + layerWeights.set_size(layerWeightSize, 1); + layerGradients.set_size(layerWeightSize, 1); +} + template WeightNormType:: WeightNormType(Layer* layer) : wrappedLayer(layer) @@ -30,6 +41,68 @@ WeightNormType(Layer* layer) : wrappedLayer(layer) layerGradients.set_size(layerWeightSize, 1); } +template +WeightNormType::WeightNormType( + const WeightNormType& other) : + wrappedLayer(other.wrappedLayer->Clone()), + layerWeightSize(other.layerWeightSize), + weights(other.weights), + layerGradients(other.layerGradients), + layerWeights(other.layerWeights) +{ + // Nothing else to do. +} + +template +WeightNormType::WeightNormType( + WeightNormType&& other) : + wrappedLayer(std::move(other.wrappedLayer)), + layerWeightSize(other.layerWeightSize), + weights(std::move(other.weights)), + layerGradients(std::move(other.layerGradients)), + layerWeights(std::move(other.layerWeights)) +{ + // Reset the other layer. + other = WeightNormType(); +} + +template +WeightNormType& +WeightNormType::operator=( + const WeightNormType& other) +{ + if (this != &other) + { + wrappedLayer = other.wrappedLayer->Clone(); + layerWeightSize = other.layerWeightSize; + weights = other.weights; + layerWeights = other.layerWeights; + layerGradients = other.layerGradients; + } + + return *this; +} + +template +WeightNormType& +WeightNormType::operator=( + WeightNormType&& other) +{ + if (this != &other) + { + wrappedLayer = std::move(other.wrappedLayer); + layerWeightSize = other.layerWeightSize; + weights = std::move(other.weights); + layerWeights = std::move(other.layerWeights); + layerGradients = std::move(other.layerGradients); + + // Reset the other layer. + other = WeightNormType(); + } + + return *this; +} + template WeightNormType::~WeightNormType() { diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 5f60264ecb..c7666d690a 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -424,9 +424,9 @@ TEST_CASE("GradientMeanSquaredErrorTest", "[LossFunctionsTest]") model = new FFN, NguyenWidrowInitialization>(); model->Predictors() = input; model->Responses() = target; - model->Add(); - model->Add(10, 2); - model->Add(); + model->Add>(); + model->Add>(10, 2); + model->Add>(); } ~GradientFunction() @@ -467,9 +467,9 @@ TEST_CASE("GradientReconstructionLossTest", "[LossFunctionsTest]") model = new FFN, NguyenWidrowInitialization>(); model->Predictors() = input; model->Responses() = target; - model->Add(); - model->Add(10, 2); - model->Add(); + model->Add>(); + model->Add>(10, 2); + model->Add>(); } ~GradientFunction()