From b4dd754ed8af983a54352cf43c2bea5716f15a78 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Fri, 28 Jul 2017 14:56:20 +0200 Subject: [PATCH 01/11] Add Fast LSTM layer implementation. --- src/mlpack/methods/ann/layer/fast_lstm.hpp | 299 ++++++++++++++++++ .../methods/ann/layer/fast_lstm_impl.hpp | 289 +++++++++++++++++ 2 files changed, 588 insertions(+) create mode 100644 src/mlpack/methods/ann/layer/fast_lstm.hpp create mode 100644 src/mlpack/methods/ann/layer/fast_lstm_impl.hpp diff --git a/src/mlpack/methods/ann/layer/fast_lstm.hpp b/src/mlpack/methods/ann/layer/fast_lstm.hpp new file mode 100644 index 0000000000..f4c026c115 --- /dev/null +++ b/src/mlpack/methods/ann/layer/fast_lstm.hpp @@ -0,0 +1,299 @@ +/** + * @file fast_lstm.hpp + * @author Marcus Edel + * + * Definition of the Fast LSTM class, which implements a Fast LSTM network + * layer. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_FAST_LSTM_HPP +#define MLPACK_METHODS_ANN_LAYER_FAST_LSTM_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * An implementation of a faster version of the LSTM network layer. Basically by + * combining the calculation of the input, forget, output gates and hidden state + * in a single step. The standard formula changes as follows: + * + * @f{eqnarray}{ + * i &=& sigmoid(W \cdot x + W \cdot h + b) + * f &=& sigmoid(W \cdot x + W \cdot h + b) + * z &=& tanh(W \cdot x + W \cdot h + b) + * c &=& f \cdot c + i \cdot z + * o &=& sigmoid(W /cdot x + W \cdot h + b) + * h &=& o \cdot tanh(c) + * @f} + * + * Note that FastLSTM network layer does not use peephole connections between + * the cell and gates. + * + * For more information, see the following. + * + * @code + * @article{Hochreiter1997, + * author = {Hochreiter, Sepp and Schmidhuber, J\"{u}rgen}, + * title = {Long Short-term Memory}, + * journal = {Neural Comput.}, + * year = {1997} + * } + * @endcode + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class FastLSTM +{ + public: + // Convenience typedefs. + typedef typename InputDataType::elem_type InputElemType; + typedef typename OutputDataType::elem_type ElemType; + + //! Create the Fast LSTM object. + FastLSTM(); + + /** + * Create the Fast LSTM layer object using the specified parameters. + * + * @param inSize The number of input units. + * @param outSize The number of output units. + * @param rho Maximum number of steps to backpropagate through time (BPTT). + */ + FastLSTM(const size_t inSize, const size_t outSize, const size_t rho); + + /** + * Ordinary feed forward pass of a neural network, evaluating the function + * f(x) by propagating the activity forward through f. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + template + void Forward(InputType&& input, OutputType&& output); + + /** + * Ordinary feed backward pass of a neural network, calculating the function + * f(x) by propagating x backwards trough f. Using the results from the feed + * forward pass. + * + * @param input The propagated input activation. + * @param gy The backpropagated error. + * @param g The calculated gradient. + */ + template + void Backward(const InputType&& /* input */, + ErrorType&& gy, + GradientType&& g); + + /* + * Reset the layer parameter. + */ + void Reset(); + + /* + * Resets the cell to accept a new input. + * This breaks the BPTT chain starts a new one. + */ + void ResetCell(); + + /* + * Calculate the gradient using the output delta and the input activation. + * + * @param input The input parameter used for calculating the gradient. + * @param error The calculated error. + * @param gradient The calculated gradient. + */ + template + void Gradient(InputType&& input, + ErrorType&& /* error */, + GradientType&& /* gradient */); + + //! Get the maximum number of steps to backpropagate through time (BPTT). + size_t Rho() const { return rho; } + //! Modify the maximum number of steps to backpropagate through time (BPTT). + size_t& Rho() { return rho; } + + //! Get the parameters. + OutputDataType const& Parameters() const { return weights; } + //! Modify the parameters. + OutputDataType& Parameters() { return weights; } + + //! Get the input parameter. + InputDataType const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } + + /** + * Serialize the layer + */ + template + void Serialize(Archive& ar, const unsigned int /* version */); + + private: + + /** + * This speeds up the sigmoid operation by using an approximation. + * + * @param input The input data. + * @param sigmoid The matrix to store the sigmoid approximation into. + */ + template + void FastSigmoid(InputType&& input, OutputType&& sigmoids) + { + #pragma omp parallel for + for (omp_size_t i = 0; i < input.n_elem; ++i) + sigmoids(i) = FastSigmoid(input(i)); + } + + /** + * Sigmoid approximation for the given sample. + * + * @param data The given data sample for the sigmoid approximation. + * @tparam The sigmoid approximation. + */ + ElemType FastSigmoid(const InputElemType data) + { + ElemType x = 0.5 * data; + ElemType z; + if (x >= 0) + { + if (x < 1.7) + z = (1.5 * x / (1 + x)); + else if (x < 3) + z = (0.935409070603099 + 0.0458812946797165 * (x - 1.7)); + else + z = 0.99505475368673; + } + else + { + ElemType xx = -x; + if (xx < 1.7) + z = (1.5 * xx / (1 + xx)); + else if (xx < 3) + z = (0.935409070603099 + 0.0458812946797165 * (xx - 1.7)); + else + z = 0.99505475368673; + z = -z; + } + + return 0.5 * (z + 1.0); + } + + //! Locally-stored number of input units. + size_t inSize; + + //! Locally-stored number of output units. + size_t outSize; + + //! Number of steps to backpropagate through time (BPTT). + size_t rho; + + //! Locally-stored number of forward steps. + size_t forwardStep; + + //! Locally-stored number of backward steps. + size_t backwardStep; + + //! Locally-stored number of gradient steps. + size_t gradientStep; + + //! Locally-stored weight object. + OutputDataType weights; + + //! Locally-stored previous output. + OutputDataType prevOutput; + + //! Locally-stored batch size. + size_t batchSize; + + //! Current batch step, alias for batchSize - 1. + size_t batchStep; + + //! Current gradient step to keep track of the backpropagate through time + //! step. + size_t gradientStepIdx; + + //! Locally-stored cell activation error. + OutputDataType cellActivationError; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Weights between the output and gate. + OutputDataType output2GateWeight; + + //! Weights between the input and gate. + OutputDataType input2GateWeight; + + //! Bias between the input and gate. + OutputDataType input2GateBias; + + //! Locally-stored gate parameter. + OutputDataType gate; + + //! Locally-stored gate activation. + OutputDataType gateActivation; + + //! Locally-stored state activation. + OutputDataType stateActivation; + + //! Locally-stored cell parameter. + OutputDataType cell; + + //! Locally-stored cell activation error. + OutputDataType cellActivation; + + //! Locally-stored foget gate error. + OutputDataType forgetGateError; + + //! Locally-stored previous error. + OutputDataType prevError; + + //! Locally-stored output parameters. + OutputDataType outParameter; +}; // class FastLSTM + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "fast_lstm_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp new file mode 100644 index 0000000000..6343b8fa6f --- /dev/null +++ b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp @@ -0,0 +1,289 @@ +/** + * @file fast_lstm_impl.hpp + * @author Marcus Edel + * + * Implementation of the Fast LSTM class, which implements a fast lstm network + * layer. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_FAST_LSTM_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_FAST_LSTM_IMPL_HPP + +// In case it hasn't yet been included. +#include "fast_lstm.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +FastLSTM::FastLSTM() +{ + // Nothing to do here. +} + +template +FastLSTM::FastLSTM( + const size_t inSize, const size_t outSize, const size_t rho) : + inSize(inSize), + outSize(outSize), + rho(rho), + forwardStep(0), + backwardStep(0), + gradientStep(0), + batchSize(0), + batchStep(0), + gradientStepIdx(0) +{ + // Weights for: input to gate layer (4 * outsize * inSize + 4 * outsize) + // and output to gate (4 * outSize). + weights.set_size( + 4 * outSize * inSize + 4 * outSize + 4 * outSize * outSize, 1); +} + +template +void FastLSTM::Reset() +{ + // Set the weight parameter for the input to gate layer (linear layer) using + // the overall layer parameter matrix. + input2GateWeight = OutputDataType(weights.memptr(), + 4 * outSize, inSize, false, false); + input2GateBias = OutputDataType(weights.memptr() + input2GateWeight.n_elem, + 4 * outSize, 1, false, false); + + // Set the weight parameter for the output to gate layer + // (linear no bias layer) using the overall layer parameter matrix. + output2GateWeight = OutputDataType(weights.memptr() + input2GateWeight.n_elem + + input2GateBias.n_elem, 4 * outSize, outSize, false, false); +} + +template +void FastLSTM::ResetCell() +{ + forwardStep = 0; + backwardStep = batchSize * rho - 1; + gradientStep = batchSize * rho - 1; + + const size_t rhoBatchSize = rho * batchSize; + if (gate.is_empty() || gate.n_cols < rhoBatchSize) + { + gate.set_size(4 * outSize, rhoBatchSize); + gateActivation.set_size(outSize * 3, rhoBatchSize); + stateActivation.set_size(outSize, rhoBatchSize); + cellActivation.set_size(outSize, rhoBatchSize); + prevError.set_size(4 * outSize, batchSize); + + if (prevOutput.is_empty()) + { + prevOutput = arma::zeros(outSize, batchSize); + cell = arma::zeros(outSize, rho * batchSize); + cellActivationError = arma::zeros(outSize, batchSize); + outParameter = arma::zeros( + outSize, (rho + 1) * batchSize); + } + else + { + // To preserve the leading zeros, recreate the object according to given + // size specifications, while preserving the elements as well as the + // layout of the elements. + prevOutput.resize(outSize, batchSize); + cell.resize(outSize, rho * batchSize); + cellActivationError.resize(outSize, batchSize); + outParameter.resize(outSize, (rho + 1) * batchSize); + } + } +} + +template +template +void FastLSTM::Forward( + InputType&& input, OutputType&& output) +{ + // Check if the batch size changed, the number of cols is defines the input + // batch size. + if (input.n_cols != batchSize) + { + batchSize = input.n_cols; + batchStep = batchSize - 1; + ResetCell(); + } + + gate.cols(forwardStep, forwardStep + batchStep) = input2GateWeight * input + + output2GateWeight * outParameter.cols( + forwardStep, forwardStep + batchStep); + gate.cols(forwardStep, forwardStep + batchStep).each_col() += input2GateBias; + + FastSigmoid(std::move( + gate.submat(0, forwardStep, 3 * outSize - 1, forwardStep + batchStep)), + std::move(gateActivation.cols(forwardStep, forwardStep + batchStep))); + + stateActivation.cols(forwardStep, forwardStep + batchStep) = arma::tanh( + gate.submat(3 * outSize, forwardStep, 4 * outSize - 1, + forwardStep + batchStep)); + + + // Update the cell: cmul1 + cmul2 + // where cmul1 is input gate * hidden state and + // cmul2 is forget gate * cell (prevCell). + if (forwardStep == 0) + { + cell.cols(forwardStep, forwardStep + batchStep) = + gateActivation.submat(0, forwardStep, outSize - 1, + forwardStep + batchStep) % + stateActivation.cols(forwardStep, forwardStep + batchStep); + } + else + { + cell.cols(forwardStep, forwardStep + batchStep) = + gateActivation.submat(0, forwardStep, outSize - 1, + forwardStep + batchStep) % + stateActivation.cols(forwardStep, forwardStep + batchStep) + + gateActivation.submat(2 * outSize, forwardStep, 3 * outSize - 1, + forwardStep + batchStep) % + cell.cols(forwardStep - batchSize, forwardStep); + } + + cellActivation.cols(forwardStep, forwardStep + batchStep) = + arma::tanh(cell.cols(forwardStep, forwardStep + batchStep)); + + outParameter.cols(forwardStep + batchSize, + forwardStep + batchSize + batchStep) = cellActivation.cols( + forwardStep, forwardStep + batchStep) % gateActivation.submat( + 1 * outSize, forwardStep, 2 * outSize - 1, forwardStep + batchStep); + + output = OutputType(outParameter.memptr() + + (forwardStep + batchSize) * outSize, outSize, batchSize, false, false); + + forwardStep += batchSize; + if ((forwardStep / batchSize) == rho) + { + forwardStep = 0; + } +} + +template +template +void FastLSTM::Backward( + const InputType&& /* input */, ErrorType&& gy, GradientType&& g) +{ + if (gradientStepIdx > 0) + { + gy += output2GateWeight.t() * prevError; + } + + cellActivationError = gy % gateActivation.submat(1 * outSize, + backwardStep - batchStep, 2 * outSize - 1, backwardStep) % + (1 - arma::pow(cellActivation.cols(backwardStep - batchStep, + backwardStep), 2)); + + if (gradientStepIdx > 0) + cellActivationError += forgetGateError; + + forgetGateError = gateActivation.submat(2 * outSize, + backwardStep - batchStep, 3 * outSize - 1, backwardStep) % + cellActivationError; + + if (backwardStep != 0) + { + prevError.submat(2 * outSize, 0, 3 * outSize - 1, batchStep) = + cell.cols((backwardStep - batchSize) - batchStep, + (backwardStep - batchSize)) % cellActivationError % + gateActivation.submat(2 * outSize, backwardStep - batchStep, + 3 * outSize - 1, backwardStep) % (1.0 - gateActivation.submat( + 2 * outSize, backwardStep - batchStep, 3 * outSize - 1, backwardStep)); + } + else + { + prevError.submat(2 * outSize, 0, 3 * outSize - 1, batchStep).zeros(); + } + + prevError.submat(0, 0,1 * outSize - 1, batchStep) = + stateActivation.cols(backwardStep - batchStep, + backwardStep) % cellActivationError % gateActivation.submat( + 0, backwardStep - batchStep, 1 * outSize - 1, backwardStep) % + (1.0 - gateActivation.submat( + 0, backwardStep - batchStep, 1 * outSize - 1, backwardStep)); + + prevError.submat(3 * outSize, 0, 4 * outSize - 1, batchStep) = + gateActivation.submat(0, backwardStep - batchStep, + outSize - 1, backwardStep) % cellActivationError % (1 - arma::pow( + stateActivation.cols(backwardStep - batchStep, backwardStep), 2)); + + prevError.submat(outSize, 0, 2 * outSize - 1, batchStep) = + cellActivation.cols(backwardStep - batchStep, + backwardStep) % gy % gateActivation.submat( + 1 * outSize, backwardStep - batchStep, 2 * outSize - 1, backwardStep) % + (1.0 - gateActivation.submat( + 1 * outSize, backwardStep - batchStep, 2 * outSize - 1, backwardStep)); + + g = input2GateWeight.t() * prevError; + + backwardStep -= batchSize; + gradientStepIdx++; + if (gradientStepIdx == rho) + { + backwardStep = rho - 1; + gradientStepIdx = 0; + } +} + +template +template +void FastLSTM::Gradient( + InputType&& input, + ErrorType&& /* error */, + GradientType&& gradient) +{ + // Gradient of the input to gate layer. + gradient.submat(0, 0, input2GateWeight.n_elem - 1, 0) = + arma::vectorise(prevError * input.t()); + gradient.submat(input2GateWeight.n_elem, 0, input2GateWeight.n_elem + + input2GateBias.n_elem - 1, 0) = arma::mean(prevError, 1); + + // Gradient of the output to gate layer. + gradient.submat(input2GateWeight.n_elem + input2GateBias.n_elem, 0, + gradient.n_elem - 1, 0) = arma::vectorise(prevError * + outParameter.cols(gradientStep - batchStep, gradientStep).t()); + + if (gradientStep == 0) + { + gradientStep = batchSize * rho - 1; + } + else + { + gradientStep -= batchSize; + } +} + +template +template +void FastLSTM::Serialize( + Archive& ar, const unsigned int /* version */) +{ + ar & data::CreateNVP(weights, "weights"); + ar & data::CreateNVP(inSize, "inSize"); + ar & data::CreateNVP(outSize, "outSize"); + ar & data::CreateNVP(rho, "rho"); + ar & data::CreateNVP(batchSize, "batchSize"); + ar & data::CreateNVP(batchStep, "batchStep"); + ar & data::CreateNVP(forwardStep, "forwardStep"); + ar & data::CreateNVP(backwardStep, "backwardStep"); + ar & data::CreateNVP(gradientStep, "gradientStep"); + ar & data::CreateNVP(gradientStepIdx, "gradientStepIdx"); + ar & data::CreateNVP(cell, "cell"); + ar & data::CreateNVP(stateActivation, "stateActivation"); + ar & data::CreateNVP(gateActivation, "gateActivation"); + ar & data::CreateNVP(gate, "gate"); + ar & data::CreateNVP(cellActivation, "cellActivation"); + ar & data::CreateNVP(forgetGateError, "forgetGateError"); + ar & data::CreateNVP(prevError, "prevError"); + ar & data::CreateNVP(outParameter, "outParameter"); +} + +} // namespace ann +} // namespace mlpack + +#endif From c40b8b444949fcbd2e635975b5856e9707c785bc Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Wed, 30 Aug 2017 17:47:06 +0200 Subject: [PATCH 02/11] Pass the current rho parameter when calling ResetCell. --- src/mlpack/methods/ann/layer/fast_lstm.hpp | 31 ++++++---- .../methods/ann/layer/fast_lstm_impl.hpp | 56 +++++++++++-------- src/mlpack/methods/ann/layer/gru.hpp | 8 ++- src/mlpack/methods/ann/layer/gru_impl.hpp | 2 +- src/mlpack/methods/ann/rnn.hpp | 3 - src/mlpack/methods/ann/rnn_impl.hpp | 4 +- .../ann/visitor/reset_cell_visitor.hpp | 11 +++- .../ann/visitor/reset_cell_visitor_impl.hpp | 12 +++- 8 files changed, 75 insertions(+), 52 deletions(-) diff --git a/src/mlpack/methods/ann/layer/fast_lstm.hpp b/src/mlpack/methods/ann/layer/fast_lstm.hpp index f4c026c115..caf4d1b7c9 100644 --- a/src/mlpack/methods/ann/layer/fast_lstm.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm.hpp @@ -14,21 +14,22 @@ #define MLPACK_METHODS_ANN_LAYER_FAST_LSTM_HPP #include +#include namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * An implementation of a faster version of the LSTM network layer. Basically by - * combining the calculation of the input, forget, output gates and hidden state - * in a single step. The standard formula changes as follows: + * An implementation of a faster version of the Fast LSTM network layer. + * Basically by combining the calculation of the input, forget, output gates + * and hidden state in a single step. The standard formula changes as follows: * * @f{eqnarray}{ * i &=& sigmoid(W \cdot x + W \cdot h + b) * f &=& sigmoid(W \cdot x + W \cdot h + b) * z &=& tanh(W \cdot x + W \cdot h + b) * c &=& f \cdot c + i \cdot z - * o &=& sigmoid(W /cdot x + W \cdot h + b) + * o &=& sigmoid(W \cdot x + W \cdot h + b) * h &=& o \cdot tanh(c) * @f} * @@ -72,7 +73,9 @@ class FastLSTM * @param outSize The number of output units. * @param rho Maximum number of steps to backpropagate through time (BPTT). */ - FastLSTM(const size_t inSize, const size_t outSize, const size_t rho); + FastLSTM(const size_t inSize, + const size_t outSize, + const size_t rho = std::numeric_limits::max()); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -94,7 +97,7 @@ class FastLSTM * @param g The calculated gradient. */ template - void Backward(const InputType&& /* input */, + void Backward(const InputType&& input, ErrorType&& gy, GradientType&& g); @@ -104,10 +107,12 @@ class FastLSTM void Reset(); /* - * Resets the cell to accept a new input. - * This breaks the BPTT chain starts a new one. + * Resets the cell to accept a new input. This breaks the BPTT chain starts a + * new one. + * + * @param size The current maximum number of steps through time. */ - void ResetCell(); + void ResetCell(const size_t size); /* * Calculate the gradient using the output delta and the input activation. @@ -118,8 +123,8 @@ class FastLSTM */ template void Gradient(InputType&& input, - ErrorType&& /* error */, - GradientType&& /* gradient */); + ErrorType&& error, + GradientType&& gradient); //! Get the maximum number of steps to backpropagate through time (BPTT). size_t Rho() const { return rho; } @@ -158,7 +163,6 @@ class FastLSTM void Serialize(Archive& ar, const unsigned int /* version */); private: - /** * This speeds up the sigmoid operation by using an approximation. * @@ -288,6 +292,9 @@ class FastLSTM //! Locally-stored output parameters. OutputDataType outParameter; + + //! Locally-stored current rho size. + size_t rhoSize; }; // class FastLSTM } // namespace ann diff --git a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp index 6343b8fa6f..26c794b0c4 100644 --- a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp @@ -36,7 +36,8 @@ FastLSTM::FastLSTM( gradientStep(0), batchSize(0), batchStep(0), - gradientStepIdx(0) + gradientStepIdx(0), + rhoSize(rho) { // Weights for: input to gate layer (4 * outsize * inSize + 4 * outsize) // and output to gate (4 * outSize). @@ -61,13 +62,22 @@ void FastLSTM::Reset() } template -void FastLSTM::ResetCell() +void FastLSTM::ResetCell(const size_t size) { - forwardStep = 0; - backwardStep = batchSize * rho - 1; - gradientStep = batchSize * rho - 1; + if (size == std::numeric_limits::max()) + return; - const size_t rhoBatchSize = rho * batchSize; + rhoSize = size; + + if (batchSize == 0) + return; + + forwardStep = 0; + gradientStepIdx = 0; + backwardStep = batchSize * size - 1; + gradientStep = batchSize * size - 1; + + const size_t rhoBatchSize = size * batchSize; if (gate.is_empty() || gate.n_cols < rhoBatchSize) { gate.set_size(4 * outSize, rhoBatchSize); @@ -79,10 +89,10 @@ void FastLSTM::ResetCell() if (prevOutput.is_empty()) { prevOutput = arma::zeros(outSize, batchSize); - cell = arma::zeros(outSize, rho * batchSize); + cell = arma::zeros(outSize, size * batchSize); cellActivationError = arma::zeros(outSize, batchSize); outParameter = arma::zeros( - outSize, (rho + 1) * batchSize); + outSize, (size + 1) * batchSize); } else { @@ -90,9 +100,9 @@ void FastLSTM::ResetCell() // size specifications, while preserving the elements as well as the // layout of the elements. prevOutput.resize(outSize, batchSize); - cell.resize(outSize, rho * batchSize); + cell.resize(outSize, size * batchSize); cellActivationError.resize(outSize, batchSize); - outParameter.resize(outSize, (rho + 1) * batchSize); + outParameter.resize(outSize, (size + 1) * batchSize); } } } @@ -108,7 +118,7 @@ void FastLSTM::Forward( { batchSize = input.n_cols; batchStep = batchSize - 1; - ResetCell(); + ResetCell(rhoSize); } gate.cols(forwardStep, forwardStep + batchStep) = input2GateWeight * input + @@ -124,7 +134,6 @@ void FastLSTM::Forward( gate.submat(3 * outSize, forwardStep, 4 * outSize - 1, forwardStep + batchStep)); - // Update the cell: cmul1 + cmul2 // where cmul1 is input gate * hidden state and // cmul2 is forget gate * cell (prevCell). @@ -143,7 +152,7 @@ void FastLSTM::Forward( stateActivation.cols(forwardStep, forwardStep + batchStep) + gateActivation.submat(2 * outSize, forwardStep, 3 * outSize - 1, forwardStep + batchStep) % - cell.cols(forwardStep - batchSize, forwardStep); + cell.cols(forwardStep - batchSize, forwardStep - batchSize + batchStep); } cellActivation.cols(forwardStep, forwardStep + batchStep) = @@ -152,7 +161,7 @@ void FastLSTM::Forward( outParameter.cols(forwardStep + batchSize, forwardStep + batchSize + batchStep) = cellActivation.cols( forwardStep, forwardStep + batchStep) % gateActivation.submat( - 1 * outSize, forwardStep, 2 * outSize - 1, forwardStep + batchStep); + outSize, forwardStep, 2 * outSize - 1, forwardStep + batchStep); output = OutputType(outParameter.memptr() + (forwardStep + batchSize) * outSize, outSize, batchSize, false, false); @@ -174,7 +183,7 @@ void FastLSTM::Backward( gy += output2GateWeight.t() * prevError; } - cellActivationError = gy % gateActivation.submat(1 * outSize, + cellActivationError = gy % gateActivation.submat(outSize, backwardStep - batchStep, 2 * outSize - 1, backwardStep) % (1 - arma::pow(cellActivation.cols(backwardStep - batchStep, backwardStep), 2)); @@ -200,12 +209,12 @@ void FastLSTM::Backward( prevError.submat(2 * outSize, 0, 3 * outSize - 1, batchStep).zeros(); } - prevError.submat(0, 0,1 * outSize - 1, batchStep) = + prevError.submat(0, 0, outSize - 1, batchStep) = stateActivation.cols(backwardStep - batchStep, backwardStep) % cellActivationError % gateActivation.submat( - 0, backwardStep - batchStep, 1 * outSize - 1, backwardStep) % + 0, backwardStep - batchStep, outSize - 1, backwardStep) % (1.0 - gateActivation.submat( - 0, backwardStep - batchStep, 1 * outSize - 1, backwardStep)); + 0, backwardStep - batchStep, outSize - 1, backwardStep)); prevError.submat(3 * outSize, 0, 4 * outSize - 1, batchStep) = gateActivation.submat(0, backwardStep - batchStep, @@ -215,9 +224,9 @@ void FastLSTM::Backward( prevError.submat(outSize, 0, 2 * outSize - 1, batchStep) = cellActivation.cols(backwardStep - batchStep, backwardStep) % gy % gateActivation.submat( - 1 * outSize, backwardStep - batchStep, 2 * outSize - 1, backwardStep) % + outSize, backwardStep - batchStep, 2 * outSize - 1, backwardStep) % (1.0 - gateActivation.submat( - 1 * outSize, backwardStep - batchStep, 2 * outSize - 1, backwardStep)); + outSize, backwardStep - batchStep, 2 * outSize - 1, backwardStep)); g = input2GateWeight.t() * prevError; @@ -233,15 +242,14 @@ void FastLSTM::Backward( template template void FastLSTM::Gradient( - InputType&& input, - ErrorType&& /* error */, - GradientType&& gradient) + InputType&& input, ErrorType&& /* error */, GradientType&& gradient) { // Gradient of the input to gate layer. gradient.submat(0, 0, input2GateWeight.n_elem - 1, 0) = arma::vectorise(prevError * input.t()); + gradient.submat(input2GateWeight.n_elem, 0, input2GateWeight.n_elem + - input2GateBias.n_elem - 1, 0) = arma::mean(prevError, 1); + input2GateBias.n_elem - 1, 0) = arma::sum(prevError, 1); // Gradient of the output to gate layer. gradient.submat(input2GateWeight.n_elem + input2GateBias.n_elem, 0, diff --git a/src/mlpack/methods/ann/layer/gru.hpp b/src/mlpack/methods/ann/layer/gru.hpp index d35c07b972..d180176da2 100644 --- a/src/mlpack/methods/ann/layer/gru.hpp +++ b/src/mlpack/methods/ann/layer/gru.hpp @@ -107,10 +107,12 @@ class GRU arma::Mat&& /* gradient */); /* - * Resets the cell to accept a new input. - * This breaks the BPTT chain starts a new one. + * Resets the cell to accept a new input. This breaks the BPTT chain starts a + * new one. + * + * @param size The current maximum number of steps through time. */ - void ResetCell(); + void ResetCell(const size_t size); //! The value of the deterministic parameter. bool Deterministic() const { return deterministic; } diff --git a/src/mlpack/methods/ann/layer/gru_impl.hpp b/src/mlpack/methods/ann/layer/gru_impl.hpp index 3eacbdb4a4..6146143b2d 100644 --- a/src/mlpack/methods/ann/layer/gru_impl.hpp +++ b/src/mlpack/methods/ann/layer/gru_impl.hpp @@ -311,7 +311,7 @@ void GRU::Gradient( } template -void GRU::ResetCell() +void GRU::ResetCell(const size_t /* size */) { outParameter.clear(); outParameter.push_back(std::move(arma::mat(allZeros.memptr(), diff --git a/src/mlpack/methods/ann/rnn.hpp b/src/mlpack/methods/ann/rnn.hpp index 7d4883e290..a10d4a2b0b 100644 --- a/src/mlpack/methods/ann/rnn.hpp +++ b/src/mlpack/methods/ann/rnn.hpp @@ -264,9 +264,6 @@ class RNN //! Number of steps to backpropagate through time (BPTT). size_t rho; - //! Number of steps to backpropagate through time (BPTT) at the previous step. - size_t prevRho; - //! Instantiated outputlayer used to evaluate the network. OutputLayerType outputLayer; diff --git a/src/mlpack/methods/ann/rnn_impl.hpp b/src/mlpack/methods/ann/rnn_impl.hpp index 445a7f02ad..41011b5f97 100644 --- a/src/mlpack/methods/ann/rnn_impl.hpp +++ b/src/mlpack/methods/ann/rnn_impl.hpp @@ -36,7 +36,6 @@ RNN::RNN( OutputLayerType outputLayer, InitializationRuleType initializeRule) : rho(rho), - prevRho(0), outputLayer(std::move(outputLayer)), initializeRule(std::move(initializeRule)), inputSize(0), @@ -59,7 +58,6 @@ RNN::RNN( OutputLayerType outputLayer, InitializationRuleType initializeRule) : rho(rho), - prevRho(0), outputLayer(std::move(outputLayer)), initializeRule(std::move(initializeRule)), inputSize(0), @@ -120,7 +118,7 @@ void RNN::ResetCells() { for (size_t i = 1; i < network.size(); ++i) { - boost::apply_visitor(ResetCellVisitor(), network[i]); + boost::apply_visitor(ResetCellVisitor(rho), network[i]); } } diff --git a/src/mlpack/methods/ann/visitor/reset_cell_visitor.hpp b/src/mlpack/methods/ann/visitor/reset_cell_visitor.hpp index f2ccf12dd0..d1fdee57f4 100644 --- a/src/mlpack/methods/ann/visitor/reset_cell_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/reset_cell_visitor.hpp @@ -26,23 +26,28 @@ namespace ann { class ResetCellVisitor : public boost::static_visitor { public: + //! Reset the cell using the given size. + ResetCellVisitor(const size_t size); + //! Execute the ResetCell() function. template void operator()(LayerType* layer) const; private: - //! Execute the ResetCell() function for a module which implements + size_t size; + + //! Execute the ResetCell() function for a module which implements //! the ResetCell() function. template typename std::enable_if< - HasResetCellCheck::value, void>::type + HasResetCellCheck::value, void>::type ResetCell(T* layer) const; //! Do not execute the Reset() function for a module which doesn't implement // the Reset() or Model() function. template typename std::enable_if< - !HasResetCellCheck::value, void>::type + !HasResetCellCheck::value, void>::type ResetCell(T* layer) const; }; diff --git a/src/mlpack/methods/ann/visitor/reset_cell_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/reset_cell_visitor_impl.hpp index 3d7e063c7d..659b9b5c9d 100644 --- a/src/mlpack/methods/ann/visitor/reset_cell_visitor_impl.hpp +++ b/src/mlpack/methods/ann/visitor/reset_cell_visitor_impl.hpp @@ -18,6 +18,12 @@ namespace mlpack { namespace ann { +//! ResetVisitor visitor class. +inline ResetCellVisitor::ResetCellVisitor(const size_t size) : size(size) +{ + /* Nothing to do here. */ +} + //! ResetVisitor visitor class. template inline void ResetCellVisitor::operator()(LayerType* layer) const @@ -27,15 +33,15 @@ inline void ResetCellVisitor::operator()(LayerType* layer) const template inline typename std::enable_if< - HasResetCellCheck::value, void>::type + HasResetCellCheck::value, void>::type ResetCellVisitor::ResetCell(T* layer) const { - layer->ResetCell(); + layer->ResetCell(size); } template inline typename std::enable_if< - !HasResetCellCheck::value, void>::type + !HasResetCellCheck::value, void>::type ResetCellVisitor::ResetCell(T* /* layer */) const { /* Nothing to do here. */ From 19c7a779e240bb7082bb85081bdb1741aaba8558 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Wed, 30 Aug 2017 17:48:47 +0200 Subject: [PATCH 03/11] Refactor LSTM layer (use peephole connections). --- src/mlpack/methods/ann/layer/lstm.hpp | 261 +++++---- src/mlpack/methods/ann/layer/lstm_impl.hpp | 602 ++++++++++++--------- 2 files changed, 514 insertions(+), 349 deletions(-) diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index f1a6e2aabe..52716c1bce 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -2,8 +2,7 @@ * @file lstm.hpp * @author Marcus Edel * - * Definition of the LSTM class, which implements a lstm network - * layer. + * Definition of the LSTM class, which implements a LSTM network layer. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -14,16 +13,8 @@ #define MLPACK_METHODS_ANN_LAYER_LSTM_HPP #include - #include -#include "../visitor/delta_visitor.hpp" -#include "../visitor/output_parameter_visitor.hpp" - -#include "layer_types.hpp" -#include "add_merge.hpp" -#include "sequential.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -33,6 +24,29 @@ namespace ann /** Artificial Neural Network. */ { * This class allows specification of the type of the activation functions used * for the gates and cells and also of the type of the function used to * initialize and update the peephole weights. + + * The implementation corresponds to the following algorithm: + * + * @f{eqnarray}{ + * i &=& sigmoid(W \cdot x + W \cdot h + W \cdot c + b) + * f &=& sigmoid(W \cdot x + W \cdot h + W \cdot c + b) + * z &=& tanh(W \cdot x + W \cdot h + b) + * c &=& f \cdot c + i \cdot z + * o &=& sigmoid(W \cdot x + W \cdot h + W \cdot c + b) + * h &=& o \cdot tanh(c) + * @f} + * + * For more information, see the following. + * + * @code + * @article{Graves2013, + * author = {Alex Graves and Abdel{-}rahman Mohamed and Geoffrey E. Hinton}, + * title = {Speech Recognition with Deep Recurrent Neural Networks}, + * journal = CoRR}, + * year = {2013}, + * url = {http://arxiv.org/abs/1303.5778}, + * } + * @endcode * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). @@ -56,8 +70,9 @@ class LSTM * @param outSize The number of output units. * @param rho Maximum number of steps to backpropagate through time (BPTT). */ - LSTM(const size_t inSize, const size_t outSize, const size_t rho = - std::numeric_limits::max()); + LSTM(const size_t inSize, + const size_t outSize, + const size_t rho = std::numeric_limits::max()); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -66,8 +81,8 @@ class LSTM * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - template - void Forward(arma::Mat&& input, arma::Mat&& output); + template + void Forward(InputType&& input, OutputType&& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -78,10 +93,23 @@ class LSTM * @param gy The backpropagated error. * @param g The calculated gradient. */ - template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + template + void Backward(const InputType&& input, + ErrorType&& gy, + GradientType&& g); + + /* + * Reset the layer parameter. + */ + void Reset(); + + /* + * Resets the cell to accept a new input. This breaks the BPTT chain starts a + * new one. + * + * @param size The current maximum number of steps through time. + */ + void ResetCell(const size_t size); /* * Calculate the gradient using the output delta and the input activation. @@ -90,21 +118,10 @@ class LSTM * @param error The calculated error. * @param gradient The calculated gradient. */ - template - void Gradient(arma::Mat&& input, - arma::Mat&& /* error */, - arma::Mat&& /* gradient */); - - /* - * Resets the cell to accept a new input. - * This breaks the BPTT chain starts a new one. - */ - void ResetCell(); - - //! The value of the deterministic parameter. - bool Deterministic() const { return deterministic; } - //! Modify the value of the deterministic parameter. - bool& Deterministic() { return deterministic; } + template + void Gradient(InputType&& input, + ErrorType&& error, + GradientType&& gradient); //! Get the maximum number of steps to backpropagate through time (BPTT). size_t Rho() const { return rho; } @@ -136,9 +153,6 @@ class LSTM //! Modify the gradient. OutputDataType& Gradient() { return gradient; } - //! Get the model modules. - std::vector& Model() { return network; } - /** * Serialize the layer */ @@ -155,51 +169,6 @@ class LSTM //! Number of steps to backpropagate through time (BPTT). size_t rho; - //! Current batch size. - size_t batchSize; - - //! Locally-stored weight object. - OutputDataType weights; - - //! Locally-stored previous output. - std::list::iterator prevOutput; - - //! Locally-stored previous cell state. - std::list::iterator prevCell; - - //! Locally-stored input 2 gate module. - LayerTypes input2GateModule; - - //! Locally-stored output 2 gate module. - LayerTypes output2GateModule; - - //! Locally-stored input gate module. - LayerTypes inputGateModule; - - //! Locally-stored hidden state module. - LayerTypes hiddenStateModule; - - //! Locally-stored forget gate module. - LayerTypes forgetGateModule; - - //! Locally-stored output gate module. - LayerTypes outputGateModule; - - //! Locally-stored cell module. - LayerTypes cellModule; - - //! Locally-stored cell activation module. - LayerTypes cellActivationModule; - - //! Locally-stored output parameter visitor. - OutputParameterVisitor outputParameterVisitor; - - //! Locally-stored delta visitor. - DeltaVisitor deltaVisitor; - - //! Locally-stored list of network modules. - std::vector network; - //! Locally-stored number of forward steps. size_t forwardStep; @@ -209,29 +178,24 @@ class LSTM //! Locally-stored number of gradient steps. size_t gradientStep; - //! Locally-stored cell parameters. - std::list cellParameter; + //! Locally-stored weight object. + OutputDataType weights; - //! Locally-stored output parameters. - std::list outParameter; + //! Locally-stored previous output. + OutputDataType prevOutput; - //! Matrix of all zeroes to initialize the output and the cell - arma::mat allZeros; + //! Locally-stored batch size. + size_t batchSize; - //! Iterator pointed to the last cell output processed by backward - std::list::iterator backIterator; + //! Current batch step, alias for batchSize - 1. + size_t batchStep; - //! Iterator pointed to the last output processed by gradient - std::list::iterator gradIterator; + //! Current gradient step to keep track of the backpropagate through time + //! step. + size_t gradientStepIdx; - //! Locally-stored previous error. - arma::mat prevError; - - //! Locally-stored foget gate error. - arma::mat forgetGateError; - - //! If true dropout and scaling is disabled, see notes above. - bool deterministic; + //! Locally-stored cell activation error. + OutputDataType cellActivationError; //! Locally-stored delta object. OutputDataType delta; @@ -244,6 +208,105 @@ class LSTM //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Weights between the output and input gate. + OutputDataType output2GateInputWeight; + + //! Weights between the input and gate. + OutputDataType input2GateInputWeight; + + //! Bias between the input and input gate. + OutputDataType input2GateInputBias; + + //! Weights between the cell and input gate. + OutputDataType cell2GateInputWeight; + + //! Weights between the output and forget gate. + OutputDataType output2GateForgetWeight; + + //! Weights between the input and gate. + OutputDataType input2GateForgetWeight; + + //! Bias between the input and gate. + OutputDataType input2GateForgetBias; + + //! Bias between the input and gate. + OutputDataType cell2GateForgetWeight; + + //! Weights between the output and gate. + OutputDataType output2GateOutputWeight; + + //! Weights between the input and gate. + OutputDataType input2GateOutputWeight; + + //! Bias between the input and gate. + OutputDataType input2GateOutputBias; + + //! Weights between cell and output gate. + OutputDataType cell2GateOutputWeight; + + //! Locally-stored input gate parameter. + OutputDataType inputGate; + + //! Locally-stored forget gate parameter. + OutputDataType forgetGate; + + //! Locally-stored hidden layer parameter. + OutputDataType hiddenLayer; + + //! Locally-stored output gate parameter. + OutputDataType outputGate; + + //! Locally-stored input gate activation. + OutputDataType inputGateActivation; + + //! Locally-stored forget gate activation. + OutputDataType forgetGateActivation; + + //! Locally-stored output gate activation. + OutputDataType outputGateActivation; + + //! Locally-stored hidden layer activation. + OutputDataType hiddenLayerActivation; + + //! Locally-stored input to hidden weight. + OutputDataType input2HiddenWeight; + + //! Locally-stored input to hidden bias. + OutputDataType input2HiddenBias; + + //! Locally-stored output to hidden weight. + OutputDataType output2HiddenWeight; + + //! Locally-stored cell parameter. + OutputDataType cell; + + //! Locally-stored cell activation error. + OutputDataType cellActivation; + + //! Locally-stored forget gate error. + OutputDataType forgetGateError; + + //! Locally-stored output gate error. + OutputDataType outputGateError; + + //! Locally-stored previous error. + OutputDataType prevError; + + //! Locally-stored output parameters. + OutputDataType outParameter; + + //! Locally-stored input cell error parameter. + arma::mat inputCellError; + + //! Locally-stored input gate error. + arma::mat inputGateError; + + //! Locally-stored hidden layer error. + arma::mat hiddenError; + + //! Locally-stored current rho size. + size_t rhoSize; }; // class LSTM } // namespace ann diff --git a/src/mlpack/methods/ann/layer/lstm_impl.hpp b/src/mlpack/methods/ann/layer/lstm_impl.hpp index 9deaf0f677..32cda04718 100644 --- a/src/mlpack/methods/ann/layer/lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/lstm_impl.hpp @@ -2,8 +2,7 @@ * @file lstm_impl.hpp * @author Marcus Edel * - * Implementation of the LSTM class, which implements a lstm network - * layer. + * Implementation of the LSTM class, which implements a lstm network layer. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -16,10 +15,6 @@ // In case it hasn't yet been included. #include "lstm.hpp" -#include "../visitor/forward_visitor.hpp" -#include "../visitor/backward_visitor.hpp" -#include "../visitor/gradient_visitor.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -31,306 +26,399 @@ LSTM::LSTM() template LSTM::LSTM( - const size_t inSize, - const size_t outSize, - const size_t rho) : + const size_t inSize, const size_t outSize, const size_t rho) : inSize(inSize), outSize(outSize), rho(rho), - batchSize(1), forwardStep(0), backwardStep(0), gradientStep(0), - deterministic(false) + batchSize(0), + batchStep(0), + gradientStepIdx(0), + rhoSize(rho) { - input2GateModule = new Linear<>(inSize, 4 * outSize); - output2GateModule = new LinearNoBias<>(outSize, 4 * outSize); - - network.push_back(input2GateModule); - network.push_back(output2GateModule); - - inputGateModule = new SigmoidLayer<>(); - hiddenStateModule = new TanHLayer<>(); - forgetGateModule = new SigmoidLayer<>(); - outputGateModule = new SigmoidLayer<>(); - - network.push_back(inputGateModule); - network.push_back(hiddenStateModule); - network.push_back(forgetGateModule); - network.push_back(outputGateModule); - - cellModule = new IdentityLayer<>(); - cellActivationModule = new TanHLayer<>(); - - network.push_back(cellModule); - network.push_back(cellActivationModule); - - prevError = arma::zeros(4 * outSize, batchSize); - - allZeros = arma::zeros(outSize, batchSize); - - outParameter.push_back(std::move(arma::mat(allZeros.memptr(), - allZeros.n_rows, allZeros.n_cols, false, false))); - - cellParameter.push_back(std::move(arma::mat(allZeros.memptr(), - allZeros.n_rows, allZeros.n_cols, false, false))); - - prevOutput = outParameter.begin(); - prevCell = cellParameter.begin(); - - backIterator = cellParameter.end(); - gradIterator = outParameter.end(); + weights.set_size(4 * outSize * inSize + 7 * outSize + + 4 * outSize * outSize, 1); } template -template -void LSTM::Forward( - arma::Mat&& input, arma::Mat&& output) +void LSTM::ResetCell(const size_t size) { - if (input.n_cols != batchSize) + if (size == std::numeric_limits::max()) + return; + + rhoSize = size; + + if (batchSize == 0) + return; + + forwardStep = 0; + gradientStepIdx = 0; + backwardStep = batchSize * size - 1; + gradientStep = batchSize * size - 1; + + const size_t rhoBatchSize = size * batchSize; + if (inputGate.is_empty() || inputGate.n_cols < rhoBatchSize) { - batchSize = input.n_cols; - prevError.resize(3 * outSize, batchSize); - } + inputGate.set_size(outSize, rhoBatchSize); + forgetGate.set_size(outSize, rhoBatchSize); + hiddenLayer.set_size(outSize, rhoBatchSize); + outputGate.set_size(outSize, rhoBatchSize); - boost::apply_visitor(ForwardVisitor(std::move(input), std::move( - boost::apply_visitor(outputParameterVisitor, input2GateModule))), - input2GateModule); + inputGateActivation.set_size(outSize, rhoBatchSize); + forgetGateActivation.set_size(outSize, rhoBatchSize); + outputGateActivation.set_size(outSize, rhoBatchSize); + hiddenLayerActivation.set_size(outSize, rhoBatchSize); - boost::apply_visitor(ForwardVisitor(std::move(*prevOutput), std::move( - boost::apply_visitor(outputParameterVisitor, output2GateModule))), - output2GateModule); + cellActivation.set_size(outSize, rhoBatchSize); + prevError.set_size(4 * outSize, batchSize); - output = boost::apply_visitor(outputParameterVisitor, input2GateModule) + - boost::apply_visitor(outputParameterVisitor, output2GateModule); - - boost::apply_visitor(ForwardVisitor(std::move(output.submat( - 0, 0, 1 * outSize - 1, batchSize - 1)), std::move(boost::apply_visitor( - outputParameterVisitor, inputGateModule))), inputGateModule); - - boost::apply_visitor(ForwardVisitor(std::move(output.submat( - 1 * outSize, 0, 2 * outSize - 1, batchSize - 1)), std::move( - boost::apply_visitor(outputParameterVisitor, hiddenStateModule))), - hiddenStateModule); - - boost::apply_visitor(ForwardVisitor(std::move(output.submat( - 2 * outSize, 0, 3 * outSize - 1, batchSize - 1)), std::move( - boost::apply_visitor(outputParameterVisitor, forgetGateModule))), - forgetGateModule); - - boost::apply_visitor(ForwardVisitor(std::move(output.submat( - 3 * outSize, 0, 4 * outSize - 1, batchSize - 1)), std::move( - boost::apply_visitor(outputParameterVisitor, outputGateModule))), - outputGateModule); - - // Update the cell (nextCell): cmul1 + cmul2 - // where cmul1 is input gate * hidden state and - // cmul2 is forget gate * cell (prevCell). - arma::mat tempPrevCell = (boost::apply_visitor(outputParameterVisitor, - inputGateModule) % boost::apply_visitor(outputParameterVisitor, - hiddenStateModule)) + (boost::apply_visitor(outputParameterVisitor, - forgetGateModule) % *prevCell); - - boost::apply_visitor(ForwardVisitor(std::move(tempPrevCell), std::move( - boost::apply_visitor(outputParameterVisitor, cellModule))), cellModule); - - boost::apply_visitor(ForwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, cellModule)), std::move(boost::apply_visitor( - outputParameterVisitor, cellActivationModule))), cellActivationModule); - - output = boost::apply_visitor(outputParameterVisitor, - cellActivationModule) % boost::apply_visitor(outputParameterVisitor, - outputGateModule); - - forwardStep++; - if (forwardStep == rho) - { - forwardStep = 0; - if (!deterministic) + if (cell.is_empty()) { - outParameter.push_back(std::move(arma::mat(allZeros.memptr(), - allZeros.n_rows, allZeros.n_cols, false, false))); - - cellParameter.push_back(std::move(arma::mat(allZeros.memptr(), - allZeros.n_rows, allZeros.n_cols, false, false))); - - prevOutput = --outParameter.end(); - prevCell = --cellParameter.end(); + cell = arma::zeros(outSize, size * batchSize); + outParameter = arma::zeros( + outSize, (size + 1) * batchSize); } else { - *prevOutput = std::move(arma::mat(allZeros.memptr(), - allZeros.n_rows, allZeros.n_cols, false, false)); - - *prevCell = std::move(arma::mat(allZeros.memptr(), - allZeros.n_rows, allZeros.n_cols, false, false)); + // To preserve the leading zeros, recreate the object according to given + // size specifications, while preserving the elements as well as the + // layout of the elements. + cell.resize(outSize, size * batchSize); + outParameter.resize(outSize, (size + 1) * batchSize); } } - else if (!deterministic) +} + +template +void LSTM::Reset() +{ + // Set the weight parameter for the output gate. + input2GateOutputWeight = OutputDataType(weights.memptr(), outSize, inSize, + false, false); + input2GateOutputBias = OutputDataType(weights.memptr() + + input2GateOutputWeight.n_elem, outSize, 1, false, false); + size_t offset = input2GateOutputWeight.n_elem + input2GateOutputBias.n_elem; + + // Set the weight parameter for the forget gate. + input2GateForgetWeight = OutputDataType(weights.memptr() + offset, + outSize, inSize, false, false); + input2GateForgetBias = OutputDataType(weights.memptr() + + offset + input2GateForgetWeight.n_elem, outSize, 1, false, false); + offset += input2GateForgetWeight.n_elem + input2GateForgetBias.n_elem; + + // Set the weight parameter for the input gate. + input2GateInputWeight = OutputDataType(weights.memptr() + + offset, outSize, inSize, false, false); + input2GateInputBias = OutputDataType(weights.memptr() + + offset + input2GateInputWeight.n_elem, outSize, 1, false, false); + offset += input2GateInputWeight.n_elem + input2GateInputBias.n_elem; + + // Set the weight parameter for the hidden gate. + input2HiddenWeight = OutputDataType(weights.memptr() + + offset, outSize, inSize, false, false); + input2HiddenBias = OutputDataType(weights.memptr() + + offset + input2HiddenWeight.n_elem, outSize, 1, false, false); + offset += input2HiddenWeight.n_elem + input2HiddenBias.n_elem; + + // Set the weight parameter for the output multiplication. + output2GateOutputWeight = OutputDataType(weights.memptr() + + offset, outSize, outSize, false, false); + offset += output2GateOutputWeight.n_elem; + + // Set the weight parameter for the output multiplication. + output2GateForgetWeight = OutputDataType(weights.memptr() + + offset, outSize, outSize, false, false); + offset += output2GateForgetWeight.n_elem; + + // Set the weight parameter for the input multiplication. + output2GateInputWeight = OutputDataType(weights.memptr() + + offset, outSize, outSize, false, false); + offset += output2GateInputWeight.n_elem; + + // Set the weight parameter for the hidden multiplication. + output2HiddenWeight = OutputDataType(weights.memptr() + + offset, outSize, outSize, false, false); + offset += output2HiddenWeight.n_elem; + + // Set the weight parameter for the cell multiplication. + cell2GateOutputWeight = OutputDataType(weights.memptr() + + offset, outSize, 1, false, false); + offset += cell2GateOutputWeight.n_elem; + + // Set the weight parameter for the cell - forget gate multiplication. + cell2GateForgetWeight = OutputDataType(weights.memptr() + + offset, outSize, 1, false, false); + offset += cell2GateOutputWeight.n_elem; + + // Set the weight parameter for the cell - input gate multiplication. + cell2GateInputWeight = OutputDataType(weights.memptr() + + offset, outSize, 1, false, false); +} + +template +template +void LSTM::Forward( + InputType&& input, OutputType&& output) +{ + // Check if the batch size changed, the number of cols is defines the input + // batch size. + if (input.n_cols != batchSize) { - outParameter.push_back(output); - cellParameter.push_back(std::move(tempPrevCell)); - prevOutput = --outParameter.end(); - prevCell = --cellParameter.end(); + batchSize = input.n_cols; + batchStep = batchSize - 1; + ResetCell(rhoSize); + } + + inputGate.cols(forwardStep, forwardStep + batchStep) = input2GateInputWeight * + input + output2GateInputWeight * outParameter.cols(forwardStep, + forwardStep + batchStep); + inputGate.cols(forwardStep, forwardStep + batchStep).each_col() += + input2GateInputBias; + + forgetGate.cols(forwardStep, forwardStep + batchStep) = input2GateForgetWeight + * input + output2GateForgetWeight * outParameter.cols( + forwardStep, forwardStep + batchStep); + forgetGate.cols(forwardStep, forwardStep + batchStep).each_col() += + input2GateForgetBias; + + if (forwardStep > 0) + { + inputGate.cols(forwardStep, forwardStep + batchStep) += + cell2GateInputWeight % cell.cols(forwardStep - batchSize, + forwardStep - batchSize + batchStep); + + forgetGate.cols(forwardStep, forwardStep + batchStep) += + cell2GateForgetWeight % cell.cols(forwardStep - batchSize, + forwardStep - batchSize + batchStep); + } + + inputGateActivation.cols(forwardStep, forwardStep + batchStep) = 1.0 / + (1 + arma::exp(-inputGate.cols(forwardStep, forwardStep + batchStep))); + + forgetGateActivation.cols(forwardStep, forwardStep + batchStep) = 1.0 / + (1 + arma::exp(-forgetGate.cols(forwardStep, forwardStep + batchStep))); + + hiddenLayer.cols(forwardStep, forwardStep + batchStep) = input2HiddenWeight * + input + output2HiddenWeight * outParameter.cols( + forwardStep, forwardStep + batchStep); + + hiddenLayer.cols(forwardStep, forwardStep + batchStep).each_col() += + input2HiddenBias; + + hiddenLayerActivation.cols(forwardStep, forwardStep + batchStep) = + arma::tanh(hiddenLayer.cols(forwardStep, forwardStep + batchStep)); + + if (forwardStep == 0) + { + cell.cols(forwardStep, forwardStep + batchStep) = + inputGateActivation.cols(forwardStep, forwardStep + batchStep) % + hiddenLayerActivation.cols(forwardStep, forwardStep + batchStep); } else { - if (forwardStep == 1) - { - outParameter.clear(); - cellParameter.clear(); + cell.cols(forwardStep, forwardStep + batchStep) = + forgetGateActivation.cols(forwardStep, forwardStep + batchStep) % + cell.cols(forwardStep - batchSize, forwardStep - batchSize + batchStep) + + inputGateActivation.cols(forwardStep, forwardStep + batchStep) % + hiddenLayerActivation.cols(forwardStep, forwardStep + batchStep); + } - outParameter.push_back(output); - cellParameter.push_back(std::move(tempPrevCell)); + outputGate.cols(forwardStep, forwardStep + batchStep) = input2GateOutputWeight + * input + output2GateOutputWeight * outParameter.cols( + forwardStep, forwardStep + batchStep) + cell.cols(forwardStep, + forwardStep + batchStep).each_col() % cell2GateOutputWeight; - prevOutput = outParameter.begin(); - prevCell = cellParameter.begin(); - } - else - { - *prevOutput = output; - *prevCell = std::move(tempPrevCell); - } + outputGate.cols(forwardStep, forwardStep + batchStep).each_col() += + input2GateOutputBias; + + outputGateActivation.cols(forwardStep, forwardStep + batchStep) = 1.0 / + (1 + arma::exp(-outputGate.cols(forwardStep, forwardStep + batchStep))); + + cellActivation.cols(forwardStep, forwardStep + batchStep) = + arma::tanh(cell.cols(forwardStep, forwardStep + batchStep)); + + outParameter.cols(forwardStep + batchSize, + forwardStep + batchSize + batchStep) = + cellActivation.cols(forwardStep, forwardStep + batchStep) % + outputGateActivation.cols(forwardStep, forwardStep + batchStep); + + output = OutputType(outParameter.memptr() + + (forwardStep + batchSize) * outSize, outSize, batchSize, false, false); + + forwardStep += batchSize; + if ((forwardStep / batchSize) == rho) + { + forwardStep = 0; } } template -template +template void LSTM::Backward( - const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) + const InputType&& /* input */, ErrorType&& gy, GradientType&& g) { - if (input.n_cols != batchSize) + if (gradientStepIdx > 0) { - batchSize = input.n_cols; - prevError.resize(3 * outSize, batchSize); + gy += prevError; } - if ((outParameter.size() - backwardStep - 1) % rho != 0 && backwardStep != 0) + outputGateError = + gy % cellActivation.cols(backwardStep - batchStep, backwardStep) % + (outputGateActivation.cols(backwardStep - batchStep, backwardStep) % + (1.0 - outputGateActivation.cols(backwardStep - batchStep, + backwardStep))); + + arma::mat cellError = gy % outputGateActivation.cols(backwardStep - batchStep, + backwardStep) % (1 - arma::pow(cellActivation.cols(backwardStep - + batchStep, backwardStep), 2)) + outputGateError.each_col() % + cell2GateOutputWeight; + + if (gradientStepIdx > 0) { - gy += boost::apply_visitor(deltaVisitor, output2GateModule); + cellError += inputCellError; } - if (backIterator == cellParameter.end()) + if (backwardStep != 0) { - backIterator = --(--cellParameter.end()); + forgetGateError = cell.cols((backwardStep - batchSize) - batchStep, + (backwardStep - batchSize)) % cellError % (forgetGateActivation.cols( + backwardStep - batchStep, backwardStep) % (1.0 - + forgetGateActivation.cols(backwardStep - batchStep, backwardStep))); + } + else + { + forgetGateError.zeros(); } - arma::mat g1 = boost::apply_visitor(outputParameterVisitor, - cellActivationModule) % gy; + inputGateError = hiddenLayerActivation.cols(backwardStep - batchStep, + backwardStep) % cellError % + (inputGateActivation.cols(backwardStep - batchStep, backwardStep) % + (1.0 - inputGateActivation.cols(backwardStep - batchStep, backwardStep))); - arma::mat g2 = boost::apply_visitor(outputParameterVisitor, - outputGateModule) % gy; + hiddenError = inputGateActivation.cols(backwardStep - batchStep, + backwardStep) % cellError % (1 - arma::pow(hiddenLayerActivation.cols( + backwardStep - batchStep, backwardStep), 2)); - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, cellActivationModule)), std::move(g2), - std::move(boost::apply_visitor(deltaVisitor, cellActivationModule))), - cellActivationModule); + inputCellError = forgetGateActivation.cols(backwardStep - batchStep, + backwardStep) %cellError + forgetGateError.each_col() % + cell2GateForgetWeight + inputGateError.each_col() % cell2GateInputWeight; - arma::mat cellActivationError = boost::apply_visitor(deltaVisitor, - cellActivationModule); + g = input2GateInputWeight.t() * inputGateError + + input2HiddenWeight.t() * hiddenError + + input2GateForgetWeight.t() * forgetGateError + + input2GateOutputWeight.t() * outputGateError; - if (backwardStep > 0) + prevError = output2GateOutputWeight.t() * outputGateError + + output2GateForgetWeight.t() * forgetGateError + + output2GateInputWeight.t() * inputGateError + + output2HiddenWeight.t() * hiddenError; + + backwardStep -= batchSize; + gradientStepIdx++; + if (gradientStepIdx == rho) { - cellActivationError += forgetGateError; + backwardStep = rho - 1; + gradientStepIdx = 0; } - - arma::mat g4 = boost::apply_visitor(outputParameterVisitor, - inputGateModule) % cellActivationError; - - arma::mat g5 = boost::apply_visitor(outputParameterVisitor, - hiddenStateModule) % cellActivationError; - - forgetGateError = boost::apply_visitor(outputParameterVisitor, - forgetGateModule) % cellActivationError; - - arma::mat g7 = *backIterator % cellActivationError; - - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, inputGateModule)), std::move(g5), - std::move(boost::apply_visitor(deltaVisitor, inputGateModule))), - inputGateModule); - - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, hiddenStateModule)), std::move(g4), - std::move(boost::apply_visitor(deltaVisitor, hiddenStateModule))), - hiddenStateModule); - - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, forgetGateModule)), std::move(g7), - std::move(boost::apply_visitor(deltaVisitor, forgetGateModule))), - forgetGateModule); - - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, outputGateModule)), std::move(g1), - std::move(boost::apply_visitor(deltaVisitor, outputGateModule))), - outputGateModule); - - prevError.submat(0, 0, 1 * outSize - 1, batchSize - 1) = boost::apply_visitor( - deltaVisitor, inputGateModule); - prevError.submat(1 * outSize, 0, 2 * outSize - 1, batchSize - 1) = - boost::apply_visitor(deltaVisitor, hiddenStateModule); - prevError.submat(2 * outSize, 0, 3 * outSize - 1, batchSize - 1) = - boost::apply_visitor(deltaVisitor, forgetGateModule); - prevError.submat(3 * outSize, 0, 4 * outSize - 1, batchSize - 1) = - boost::apply_visitor(deltaVisitor, outputGateModule); - - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, input2GateModule)), std::move(prevError), - std::move(boost::apply_visitor(deltaVisitor, input2GateModule))), - input2GateModule); - - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, output2GateModule)), std::move(prevError), - std::move(boost::apply_visitor(deltaVisitor, output2GateModule))), - output2GateModule); - - backwardStep++; - backIterator--; - - g = boost::apply_visitor(deltaVisitor, input2GateModule); } template -template +template void LSTM::Gradient( - arma::Mat&& input, - arma::Mat&& /* error */, - arma::Mat&& /* gradient */) + InputType&& input, ErrorType&& /* error */, GradientType&& gradient) { - if (gradIterator == outParameter.end()) + // Input2GateOutputWeight and input2GateOutputBias gradients. + gradient.submat(0, 0, input2GateOutputWeight.n_elem - 1, 0) = + arma::vectorise(outputGateError * input.t()); + gradient.submat(input2GateOutputWeight.n_elem, 0, + input2GateOutputWeight.n_elem + input2GateOutputBias.n_elem - 1, 0) = + arma::sum(outputGateError, 1); + size_t offset = input2GateOutputWeight.n_elem + input2GateOutputBias.n_elem; + + // Input2GateForgetWeight and input2GateForgetBias gradients. + gradient.submat(offset, 0, offset + input2GateForgetWeight.n_elem - 1, 0) = + arma::vectorise(forgetGateError * input.t()); + gradient.submat(offset + input2GateForgetWeight.n_elem, 0, + offset + input2GateForgetWeight.n_elem + + input2GateForgetBias.n_elem - 1, 0) = arma::sum(forgetGateError, 1); + offset += input2GateForgetWeight.n_elem + input2GateForgetBias.n_elem; + + // Input2GateInputWeight and input2GateInputBias gradients. + gradient.submat(offset, 0, offset + input2GateInputWeight.n_elem - 1, 0) = + arma::vectorise(inputGateError * input.t()); + gradient.submat(offset + input2GateInputWeight.n_elem, 0, + offset + input2GateInputWeight.n_elem + + input2GateInputBias.n_elem - 1, 0) = arma::sum(inputGateError, 1); + offset += input2GateInputWeight.n_elem + input2GateInputBias.n_elem; + + // Input2HiddenWeight and input2HiddenBias gradients. + gradient.submat(offset, 0, offset + input2HiddenWeight.n_elem - 1, 0) = + arma::vectorise(hiddenError * input.t()); + gradient.submat(offset + input2HiddenWeight.n_elem, 0, + offset + input2HiddenWeight.n_elem + input2HiddenBias.n_elem - 1, 0) = + arma::sum(hiddenError, 1); + offset += input2HiddenWeight.n_elem + input2HiddenBias.n_elem; + + // Output2GateOutputWeight gradients. + gradient.submat(offset, 0, offset + output2GateOutputWeight.n_elem - 1, 0) = + arma::vectorise(outputGateError * + outParameter.cols(gradientStep - batchStep, gradientStep).t()); + offset += output2GateOutputWeight.n_elem; + + // Output2GateForgetWeight gradients. + gradient.submat(offset, 0, offset + output2GateForgetWeight.n_elem - 1, 0) = + arma::vectorise(forgetGateError * + outParameter.cols(gradientStep - batchStep, gradientStep).t()); + offset += output2GateForgetWeight.n_elem; + + // Output2GateInputWeight gradients. + gradient.submat(offset, 0, offset + output2GateInputWeight.n_elem - 1, 0) = + arma::vectorise(inputGateError * + outParameter.cols(gradientStep - batchStep, gradientStep).t()); + offset += output2GateInputWeight.n_elem; + + // Output2HiddenWeight gradients. + gradient.submat(offset, 0, offset + output2HiddenWeight.n_elem - 1, 0) = + arma::vectorise(hiddenError * + outParameter.cols(gradientStep - batchStep, gradientStep).t()); + offset += output2HiddenWeight.n_elem; + + // Cell2GateOutputWeight gradients. + gradient.submat(offset, 0, offset + cell2GateOutputWeight.n_elem - 1, 0) = + arma::sum(outputGateError % + cell.cols(gradientStep - batchStep, gradientStep), 1); + offset += cell2GateOutputWeight.n_elem; + + // Cell2GateForgetWeight and cell2GateInputWeight gradients. + if (gradientStep != 0) { - gradIterator = --(--outParameter.end()); + gradient.submat(offset, 0, offset + cell2GateForgetWeight.n_elem - 1, 0) = + arma::sum(forgetGateError % cell.cols(gradientStep - batchStep - + batchSize, gradientStep - batchSize), 1); + gradient.submat(offset + cell2GateForgetWeight.n_elem, 0, offset + + cell2GateForgetWeight.n_elem + cell2GateInputWeight.n_elem - 1, 0) = + arma::sum(inputGateError % cell.cols(gradientStep - batchStep - + batchSize, gradientStep - batchSize), 1); + } + else + { + gradient.submat(offset, 0, offset + + cell2GateForgetWeight.n_elem - 1, 0).zeros(); + gradient.submat(offset + cell2GateForgetWeight.n_elem, 0, offset + + cell2GateForgetWeight.n_elem + + cell2GateInputWeight.n_elem - 1, 0).zeros(); } - boost::apply_visitor(GradientVisitor(std::move(input), std::move(prevError)), - input2GateModule); - - boost::apply_visitor(GradientVisitor( - std::move(*gradIterator), - std::move(prevError)), output2GateModule); - - gradIterator--; -} - -template -void LSTM::ResetCell() -{ - outParameter.clear(); - outParameter.push_back(std::move(arma::mat(allZeros.memptr(), - allZeros.n_rows, allZeros.n_cols, false, false))); - - cellParameter.clear(); - cellParameter.push_back(std::move(arma::mat(allZeros.memptr(), - allZeros.n_rows, allZeros.n_cols, false, false))); - - prevOutput = outParameter.begin(); - prevCell = cellParameter.begin(); - - backIterator = cellParameter.end(); - gradIterator = outParameter.end(); - - forwardStep = 0; - backwardStep = 0; + if (gradientStep == 0) + { + gradientStep = batchSize * rho - 1; + } + else + { + gradientStep -= batchSize; + } } template @@ -342,6 +430,20 @@ void LSTM::Serialize( ar & data::CreateNVP(inSize, "inSize"); ar & data::CreateNVP(outSize, "outSize"); ar & data::CreateNVP(rho, "rho"); + ar & data::CreateNVP(batchSize, "batchSize"); + ar & data::CreateNVP(batchStep, "batchStep"); + ar & data::CreateNVP(forwardStep, "forwardStep"); + ar & data::CreateNVP(backwardStep, "backwardStep"); + ar & data::CreateNVP(gradientStep, "gradientStep"); + ar & data::CreateNVP(gradientStepIdx, "gradientStepIdx"); + ar & data::CreateNVP(cell, "cell"); + ar & data::CreateNVP(inputGateActivation, "inputGateActivation"); + ar & data::CreateNVP(forgetGateActivation, "forgetGateActivation"); + ar & data::CreateNVP(outputGateActivation, "outputGateActivation"); + ar & data::CreateNVP(hiddenLayerActivation, "hiddenLayerActivation"); + ar & data::CreateNVP(cellActivation, "cellActivation"); + ar & data::CreateNVP(prevError, "prevError"); + ar & data::CreateNVP(outParameter, "outParameter"); } } // namespace ann From ff373067504c931f1492a37fc385a236fd405d32 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Wed, 30 Aug 2017 17:57:38 +0200 Subject: [PATCH 04/11] Test the FastLSTM layer and optimize the recurrent network test suite. --- src/mlpack/methods/ann/layer/CMakeLists.txt | 2 + src/mlpack/methods/ann/layer/layer.hpp | 1 + src/mlpack/methods/ann/layer/layer_types.hpp | 2 + src/mlpack/tests/ann_layer_test.cpp | 49 ++++++++++- src/mlpack/tests/recurrent_network_test.cpp | 89 ++++++++++---------- 5 files changed, 98 insertions(+), 45 deletions(-) diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt index 5c8ee2c3b0..1229781cbe 100644 --- a/src/mlpack/methods/ann/layer/CMakeLists.txt +++ b/src/mlpack/methods/ann/layer/CMakeLists.txt @@ -22,6 +22,8 @@ set(SOURCES dropout_impl.hpp elu.hpp elu_impl.hpp + fast_lstm.hpp + fast_lstm_impl.hpp glimpse.hpp glimpse_impl.hpp gru.hpp diff --git a/src/mlpack/methods/ann/layer/layer.hpp b/src/mlpack/methods/ann/layer/layer.hpp index 6f0fbc8c1d..f47eadfd2f 100644 --- a/src/mlpack/methods/ann/layer/layer.hpp +++ b/src/mlpack/methods/ann/layer/layer.hpp @@ -22,6 +22,7 @@ #include "linear_no_bias.hpp" #include "lstm.hpp" #include "gru.hpp" +#include "fast_lstm.hpp" #include "recurrent.hpp" #include "recurrent_attention.hpp" #include "sequential.hpp" diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 6ea36e3339..ddf377a84d 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -51,6 +51,7 @@ template class Linear; template class LinearNoBias; template class LSTM; template class GRU; +template class FastLSTM; template class Recurrent; template class Sequential; template class VRClassReward; @@ -105,6 +106,7 @@ using LayerTypes = boost::variant< Lookup*, LSTM*, GRU*, + FastLSTM*, MaxPooling*, MeanPooling*, MeanSquaredError*, diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 74224f67d2..70acc5d915 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -152,7 +152,7 @@ double CheckGradient(FunctionType& function, const double eps = 1e-7) estGradient = arma::zeros(orgGradient.n_rows, orgGradient.n_cols); - // compute numeric approximations to gradient. + // Compute numeric approximations to gradient. for (size_t i = 0; i < orgGradient.n_elem; ++i) { double tmp = function.Parameters()(i); @@ -779,6 +779,52 @@ BOOST_AUTO_TEST_CASE(GradientLSTMLayerTest) BOOST_REQUIRE_LE(CheckGradient(function), 1e-4); } +/** + * FastLSTM layer numerically gradient test. + */ +BOOST_AUTO_TEST_CASE(GradientFastLSTMLayerTest) +{ + // Fast LSTM function gradient instantiation. + struct GradientFunction + { + GradientFunction() + { + input = arma::randu(5, 1); + target = arma::mat("1; 1; 1; 1; 1"); + const size_t rho = 5; + + model = new RNN >(input, target, rho); + model->Add >(); + model->Add >(1, 10); + model->Add >(10, 3, rho); + model->Add >(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + arma::mat output; + double error = model->Evaluate(model->Parameters(), 0); + model->Gradient(model->Parameters(), 0, gradient); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + RNN >* model; + arma::mat input, target; + } function; + + // The threshold should be << 0.1 but since the Fast LSTM layer uses an + // approximation of the sigmoid function the estimated gradient is not + // correct. + BOOST_REQUIRE_LE(CheckGradient(function), 0.2); +} + /** * Check if the gradients computed by GRU cell are close enough to the * approximation of the gradients. @@ -1103,5 +1149,4 @@ BOOST_AUTO_TEST_CASE(SimpleMeanSquaredErrorLayerTest) BOOST_REQUIRE_EQUAL(output.n_elem, 1); } - BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp index 260501cfa3..94a6cebc72 100644 --- a/src/mlpack/tests/recurrent_network_test.cpp +++ b/src/mlpack/tests/recurrent_network_test.cpp @@ -40,7 +40,7 @@ void GenerateNoisySines(arma::mat& data, const size_t sequences, const double noise = 0.3) { - arma::colvec x = arma::linspace>(0, + arma::colvec x = arma::linspace >(0, points - 1, points) / points * 20.0; arma::colvec y1 = arma::sin(x + arma::as_scalar(arma::randu(1)) * 3.0); arma::colvec y2 = arma::sin(x / 2.0 + arma::as_scalar(arma::randu(1)) * 3.0); @@ -362,11 +362,11 @@ void GenerateNextRecursiveReber(const arma::Mat& transitions, * Train the specified network and the construct a Reber grammar dataset. */ template -void ReberGrammarTestNetwork(size_t hiddenSize = 4, - bool recursive = false, - size_t averageRecursion = 3, - size_t maxRecursion = 5 - ) +void ReberGrammarTestNetwork(const size_t hiddenSize = 4, + const bool recursive = false, + const size_t averageRecursion = 3, + const size_t maxRecursion = 5, + const size_t iterations = 10) { // Reber state transition matrix. (The last two columns are the indices to the // next path). @@ -378,8 +378,8 @@ void ReberGrammarTestNetwork(size_t hiddenSize = 4, << 'P' << 'V' << '3' << '5' << arma::endr << 'E' << 'E' << '0' << '0' << arma::endr; - const size_t trainReberGrammarCount = 800; - const size_t testReberGrammarCount = 400; + const size_t trainReberGrammarCount = 700; + const size_t testReberGrammarCount = 250; std::string trainReber, testReber; arma::field trainInput(1, trainReberGrammarCount); @@ -449,16 +449,15 @@ void ReberGrammarTestNetwork(size_t hiddenSize = 4, const size_t inputSize = 7; RNN > model(5); - model.Add >(inputSize, hiddenSize); model.Add(hiddenSize, hiddenSize); model.Add >(hiddenSize, outputSize); model.Add >(); - StandardSGD opt(0.01, 2, -50000); + MomentumSGD opt(0.06, 2, -50000); arma::mat inputTemp, labelsTemp; - for (size_t i = 0; i < (10 + offset); i++) + for (size_t i = 0; i < (iterations + offset); i++) { for (size_t j = 0; j < trainReberGrammarCount; j++) { @@ -467,6 +466,7 @@ void ReberGrammarTestNetwork(size_t hiddenSize = 4, model.Rho() = inputTemp.n_elem / inputSize; model.Train(inputTemp, labelsTemp, opt); + opt.ResetPolicy() = false; } } @@ -517,7 +517,7 @@ void ReberGrammarTestNetwork(size_t hiddenSize = 4, } error /= testReberGrammarCount; - if (error <= 0.2) + if (error <= 0.3) { ++successes; break; @@ -530,27 +530,19 @@ void ReberGrammarTestNetwork(size_t hiddenSize = 4, } /** - * Train the specified networks on a Reber grammar dataset. + * Train the specified networks on an embedded Reber grammar dataset. */ -BOOST_AUTO_TEST_CASE(LSTMReberGrammarTest) +BOOST_AUTO_TEST_CASE(LSTMRecursiveReberGrammarTest) { - ReberGrammarTestNetwork>(4, false); + ReberGrammarTestNetwork >(20, true, 3, 5, 12); } /** * Train the specified networks on an embedded Reber grammar dataset. */ -BOOST_AUTO_TEST_CASE(LSTMRecursiveReberGrammarTest) +BOOST_AUTO_TEST_CASE(FastLSTMRecursiveReberGrammarTest) { - ReberGrammarTestNetwork>(20, true); -} - -/** - * Train the specified networks on a Reber grammar dataset. - */ -BOOST_AUTO_TEST_CASE(GRUReberGrammarTest) -{ - ReberGrammarTestNetwork>(4, false); + ReberGrammarTestNetwork >(25, true); } /** @@ -558,7 +550,7 @@ BOOST_AUTO_TEST_CASE(GRUReberGrammarTest) */ BOOST_AUTO_TEST_CASE(GRURecursiveReberGrammarTest) { - ReberGrammarTestNetwork>(20, true); + ReberGrammarTestNetwork >(16, true); } /* @@ -622,10 +614,11 @@ void GenerateDistractedSequence(arma::mat& input, arma::mat& output) * dataset. */ template -void DistractedSequenceRecallTestNetwork() +void DistractedSequenceRecallTestNetwork( + const size_t cellSize, const size_t hiddenSize) { - const size_t trainDistractedSequenceCount = 800; - const size_t testDistractedSequenceCount = 400; + const size_t trainDistractedSequenceCount = 600; + const size_t testDistractedSequenceCount = 300; arma::field trainInput(1, trainDistractedSequenceCount); arma::field trainLabels(1, trainDistractedSequenceCount); @@ -645,13 +638,13 @@ void DistractedSequenceRecallTestNetwork() * output units. The hidden layer is connected to itself. The network * structure looks like: * - * Input Hidden Output - * Layer(10) Layer(layerSize) Layer(3) - * +-----+ +-----+ +-----+ - * | | | | | | - * | +------>| +------>| | - * | | ..>| | | | - * +-----+ . +--+--+ +-----+ + * Input Recurrent Hidden Output + * Layer(10) Layer(cellSize) Layer(3) Layer(3) + * +-----+ +-----+ +-----+ +-----+ + * | | | | | | | | + * | +------>| +------>| |------>| | + * | | ..>| | | | | | + * +-----+ . +--+--+ +-----+ +-----+ * . . * . . * ....... @@ -670,15 +663,17 @@ void DistractedSequenceRecallTestNetwork() { RNN > model(rho); model.Add >(); - model.Add >(inputSize, 14); - model.Add(14, 7); - model.Add >(7, outputSize); + model.Add >(inputSize, cellSize); + model.Add(cellSize, hiddenSize); + model.Add >(hiddenSize, outputSize); model.Add >(); StandardSGD opt(0.1, 2, -50000); + // We increase the number of iterations (training) if the first run didn't + // pass. arma::mat inputTemp, labelsTemp; - for (size_t i = 0; i < (10 + offset); i++) + for (size_t i = 0; i < (9 + offset); i++) { for (size_t j = 0; j < trainDistractedSequenceCount; j++) { @@ -706,7 +701,6 @@ void DistractedSequenceRecallTestNetwork() } error /= testDistractedSequenceCount; - // Can we reproduce the results from the paper. They provide an 95% accuracy // on a test set of 1000 randomly selected sequences. // Ensure that this is within tolerance, which is at least as good as the @@ -729,7 +723,16 @@ void DistractedSequenceRecallTestNetwork() */ BOOST_AUTO_TEST_CASE(LSTMDistractedSequenceRecallTest) { - DistractedSequenceRecallTestNetwork>(); + DistractedSequenceRecallTestNetwork >(4, 8); +} + +/** + * Train the specified networks on the Derek D. Monner's distracted sequence + * recall task. + */ +BOOST_AUTO_TEST_CASE(FastLSTMDistractedSequenceRecallTest) +{ + DistractedSequenceRecallTestNetwork >(4, 8); } /** @@ -738,7 +741,7 @@ BOOST_AUTO_TEST_CASE(LSTMDistractedSequenceRecallTest) */ BOOST_AUTO_TEST_CASE(GRUDistractedSequenceRecallTest) { - DistractedSequenceRecallTestNetwork>(); + DistractedSequenceRecallTestNetwork >(4, 8); } BOOST_AUTO_TEST_SUITE_END(); From e1845567aa2965eaa7f583eebf664c5a5dbdf90e Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 4 Sep 2017 16:00:22 +0200 Subject: [PATCH 05/11] Adresse static analysis issues: naming confusion, simplify linspace expression. --- src/mlpack/methods/ann/layer/fast_lstm.hpp | 6 +++--- src/mlpack/methods/ann/layer/lstm.hpp | 6 +++--- src/mlpack/tests/recurrent_network_test.cpp | 9 ++++----- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/mlpack/methods/ann/layer/fast_lstm.hpp b/src/mlpack/methods/ann/layer/fast_lstm.hpp index caf4d1b7c9..c0b3728c03 100644 --- a/src/mlpack/methods/ann/layer/fast_lstm.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm.hpp @@ -152,9 +152,9 @@ class FastLSTM OutputDataType& Delta() { return delta; } //! Get the gradient. - OutputDataType const& Gradient() const { return gradient; } + OutputDataType const& Gradient() const { return grad; } //! Modify the gradient. - OutputDataType& Gradient() { return gradient; } + OutputDataType& Gradient() { return grad; } /** * Serialize the layer @@ -252,7 +252,7 @@ class FastLSTM OutputDataType delta; //! Locally-stored gradient object. - OutputDataType gradient; + OutputDataType grad; //! Locally-stored input parameter object. InputDataType inputParameter; diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index 52716c1bce..0f55f42820 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -149,9 +149,9 @@ class LSTM OutputDataType& Delta() { return delta; } //! Get the gradient. - OutputDataType const& Gradient() const { return gradient; } + OutputDataType const& Gradient() const { return grad; } //! Modify the gradient. - OutputDataType& Gradient() { return gradient; } + OutputDataType& Gradient() { return grad; } /** * Serialize the layer @@ -201,7 +201,7 @@ class LSTM OutputDataType delta; //! Locally-stored gradient object. - OutputDataType gradient; + OutputDataType grad; //! Locally-stored input parameter object. InputDataType inputParameter; diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp index 94a6cebc72..ed962fc8f3 100644 --- a/src/mlpack/tests/recurrent_network_test.cpp +++ b/src/mlpack/tests/recurrent_network_test.cpp @@ -40,8 +40,8 @@ void GenerateNoisySines(arma::mat& data, const size_t sequences, const double noise = 0.3) { - arma::colvec x = arma::linspace >(0, - points - 1, points) / points * 20.0; + arma::colvec x = arma::linspace(0, points - 1, points) / + points * 20.0; arma::colvec y1 = arma::sin(x + arma::as_scalar(arma::randu(1)) * 3.0); arma::colvec y2 = arma::sin(x / 2.0 + arma::as_scalar(arma::randu(1)) * 3.0); @@ -534,7 +534,7 @@ void ReberGrammarTestNetwork(const size_t hiddenSize = 4, */ BOOST_AUTO_TEST_CASE(LSTMRecursiveReberGrammarTest) { - ReberGrammarTestNetwork >(20, true, 3, 5, 12); + ReberGrammarTestNetwork >(10, false); } /** @@ -586,8 +586,7 @@ void GenerateDistractedSequence(arma::mat& input, arma::mat& output) input = arma::zeros(10, 10); output = arma::zeros(3, 10); - arma::Col index = arma::shuffle(arma::linspace >( - 0, 7, 8)); + arma::uvec index = arma::shuffle(arma::linspace(0, 7, 8)); // Set the target in the input sequence and the corresponding targets in the // output sequence by following the correct order. From 63a07ce314ea700e33c39330250032cca8996c6f Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Thu, 14 Sep 2017 20:32:07 +0200 Subject: [PATCH 06/11] Remove OpenMP loop, optimize FastLSTM network test settings. --- src/mlpack/methods/ann/layer/fast_lstm.hpp | 10 ++++------ src/mlpack/tests/recurrent_network_test.cpp | 6 +++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/mlpack/methods/ann/layer/fast_lstm.hpp b/src/mlpack/methods/ann/layer/fast_lstm.hpp index c0b3728c03..57b5f21a8a 100644 --- a/src/mlpack/methods/ann/layer/fast_lstm.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm.hpp @@ -172,8 +172,7 @@ class FastLSTM template void FastSigmoid(InputType&& input, OutputType&& sigmoids) { - #pragma omp parallel for - for (omp_size_t i = 0; i < input.n_elem; ++i) + for (size_t i = 0; i < input.n_elem; ++i) sigmoids(i) = FastSigmoid(input(i)); } @@ -200,12 +199,11 @@ class FastLSTM { ElemType xx = -x; if (xx < 1.7) - z = (1.5 * xx / (1 + xx)); + z = -(1.5 * xx / (1 + xx)); else if (xx < 3) - z = (0.935409070603099 + 0.0458812946797165 * (xx - 1.7)); + z = -(0.935409070603099 + 0.0458812946797165 * (xx - 1.7)); else - z = 0.99505475368673; - z = -z; + z = -0.99505475368673; } return 0.5 * (z + 1.0); diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp index ed962fc8f3..e28ae23717 100644 --- a/src/mlpack/tests/recurrent_network_test.cpp +++ b/src/mlpack/tests/recurrent_network_test.cpp @@ -532,7 +532,7 @@ void ReberGrammarTestNetwork(const size_t hiddenSize = 4, /** * Train the specified networks on an embedded Reber grammar dataset. */ -BOOST_AUTO_TEST_CASE(LSTMRecursiveReberGrammarTest) +BOOST_AUTO_TEST_CASE(LSTMReberGrammarTest) { ReberGrammarTestNetwork >(10, false); } @@ -540,9 +540,9 @@ BOOST_AUTO_TEST_CASE(LSTMRecursiveReberGrammarTest) /** * Train the specified networks on an embedded Reber grammar dataset. */ -BOOST_AUTO_TEST_CASE(FastLSTMRecursiveReberGrammarTest) +BOOST_AUTO_TEST_CASE(FastLSTMReberGrammarTest) { - ReberGrammarTestNetwork >(25, true); + ReberGrammarTestNetwork >(8, false); } /** From 7f507e1dc13efa17bbde9e28f68b3059b11f6779 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Thu, 14 Sep 2017 22:35:02 +0200 Subject: [PATCH 07/11] Update serialize code; for more information take a look at: #1103. --- src/mlpack/methods/ann/layer/fast_lstm.hpp | 2 +- .../methods/ann/layer/fast_lstm_impl.hpp | 38 +++++++++---------- src/mlpack/methods/ann/layer/lstm.hpp | 2 +- src/mlpack/methods/ann/layer/lstm_impl.hpp | 2 +- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/mlpack/methods/ann/layer/fast_lstm.hpp b/src/mlpack/methods/ann/layer/fast_lstm.hpp index 57b5f21a8a..915f3259c5 100644 --- a/src/mlpack/methods/ann/layer/fast_lstm.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm.hpp @@ -160,7 +160,7 @@ class FastLSTM * Serialize the layer */ template - void Serialize(Archive& ar, const unsigned int /* version */); + void serialize(Archive& ar, const unsigned int /* version */); private: /** diff --git a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp index 26c794b0c4..f0256a22e6 100644 --- a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp @@ -268,27 +268,27 @@ void FastLSTM::Gradient( template template -void FastLSTM::Serialize( +void FastLSTM::serialize( Archive& ar, const unsigned int /* version */) { - ar & data::CreateNVP(weights, "weights"); - ar & data::CreateNVP(inSize, "inSize"); - ar & data::CreateNVP(outSize, "outSize"); - ar & data::CreateNVP(rho, "rho"); - ar & data::CreateNVP(batchSize, "batchSize"); - ar & data::CreateNVP(batchStep, "batchStep"); - ar & data::CreateNVP(forwardStep, "forwardStep"); - ar & data::CreateNVP(backwardStep, "backwardStep"); - ar & data::CreateNVP(gradientStep, "gradientStep"); - ar & data::CreateNVP(gradientStepIdx, "gradientStepIdx"); - ar & data::CreateNVP(cell, "cell"); - ar & data::CreateNVP(stateActivation, "stateActivation"); - ar & data::CreateNVP(gateActivation, "gateActivation"); - ar & data::CreateNVP(gate, "gate"); - ar & data::CreateNVP(cellActivation, "cellActivation"); - ar & data::CreateNVP(forgetGateError, "forgetGateError"); - ar & data::CreateNVP(prevError, "prevError"); - ar & data::CreateNVP(outParameter, "outParameter"); + ar & BOOST_SERIALIZATION_NVP(weights); + ar & BOOST_SERIALIZATION_NVP(inSize); + ar & BOOST_SERIALIZATION_NVP(outSize); + ar & BOOST_SERIALIZATION_NVP(rho); + ar & BOOST_SERIALIZATION_NVP(batchSize); + ar & BOOST_SERIALIZATION_NVP(batchStep); + ar & BOOST_SERIALIZATION_NVP(forwardStep); + ar & BOOST_SERIALIZATION_NVP(backwardStep); + ar & BOOST_SERIALIZATION_NVP(gradientStep); + ar & BOOST_SERIALIZATION_NVP(gradientStepIdx); + ar & BOOST_SERIALIZATION_NVP(cell); + ar & BOOST_SERIALIZATION_NVP(stateActivation); + ar & BOOST_SERIALIZATION_NVP(gateActivation); + ar & BOOST_SERIALIZATION_NVP(gate); + ar & BOOST_SERIALIZATION_NVP(cellActivation); + ar & BOOST_SERIALIZATION_NVP(forgetGateError); + ar & BOOST_SERIALIZATION_NVP(prevError); + ar & BOOST_SERIALIZATION_NVP(outParameter); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index 34d89ec119..800eac0d58 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -157,7 +157,7 @@ class LSTM * Serialize the layer */ template - void Serialize(Archive& ar, const unsigned int /* version */); + void serialize(Archive& ar, const unsigned int /* version */); private: //! Locally-stored number of input units. diff --git a/src/mlpack/methods/ann/layer/lstm_impl.hpp b/src/mlpack/methods/ann/layer/lstm_impl.hpp index b920455ba0..9e50b205f3 100644 --- a/src/mlpack/methods/ann/layer/lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/lstm_impl.hpp @@ -424,7 +424,7 @@ void LSTM::Gradient( template template -void LSTM::Serialize( +void LSTM::serialize( Archive& ar, const unsigned int /* version */) { ar & BOOST_SERIALIZATION_NVP(weights); From b0c30e81029521635607f48016d4fe88488ce0af Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Thu, 5 Oct 2017 01:22:38 +0200 Subject: [PATCH 08/11] Catch issue if the rho size isn't defined. --- src/mlpack/methods/ann/layer/fast_lstm.hpp | 3 + .../methods/ann/layer/fast_lstm_impl.hpp | 13 ++-- src/mlpack/methods/ann/layer/lstm.hpp | 3 + src/mlpack/methods/ann/layer/lstm_impl.hpp | 13 ++-- src/mlpack/methods/ann/rnn.hpp | 10 +-- src/mlpack/tests/ann_layer_test.cpp | 74 +++++++++++++++++++ 6 files changed, 101 insertions(+), 15 deletions(-) diff --git a/src/mlpack/methods/ann/layer/fast_lstm.hpp b/src/mlpack/methods/ann/layer/fast_lstm.hpp index 915f3259c5..3ded18274a 100644 --- a/src/mlpack/methods/ann/layer/fast_lstm.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm.hpp @@ -293,6 +293,9 @@ class FastLSTM //! Locally-stored current rho size. size_t rhoSize; + + //! Current backpropagate through time steps. + size_t bpttSteps; }; // class FastLSTM } // namespace ann diff --git a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp index f0256a22e6..3222275a4d 100644 --- a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp @@ -37,7 +37,8 @@ FastLSTM::FastLSTM( batchSize(0), batchStep(0), gradientStepIdx(0), - rhoSize(rho) + rhoSize(rho), + bpttSteps(0) { // Weights for: input to gate layer (4 * outsize * inSize + 4 * outsize) // and output to gate (4 * outSize). @@ -72,6 +73,7 @@ void FastLSTM::ResetCell(const size_t size) if (batchSize == 0) return; + bpttSteps = std::min(rho, rhoSize); forwardStep = 0; gradientStepIdx = 0; backwardStep = batchSize * size - 1; @@ -167,7 +169,7 @@ void FastLSTM::Forward( (forwardStep + batchSize) * outSize, outSize, batchSize, false, false); forwardStep += batchSize; - if ((forwardStep / batchSize) == rho) + if ((forwardStep / batchSize) == bpttSteps) { forwardStep = 0; } @@ -232,9 +234,9 @@ void FastLSTM::Backward( backwardStep -= batchSize; gradientStepIdx++; - if (gradientStepIdx == rho) + if (gradientStepIdx == bpttSteps) { - backwardStep = rho - 1; + backwardStep = bpttSteps - 1; gradientStepIdx = 0; } } @@ -258,7 +260,7 @@ void FastLSTM::Gradient( if (gradientStep == 0) { - gradientStep = batchSize * rho - 1; + gradientStep = batchSize * bpttSteps - 1; } else { @@ -275,6 +277,7 @@ void FastLSTM::serialize( ar & BOOST_SERIALIZATION_NVP(inSize); ar & BOOST_SERIALIZATION_NVP(outSize); ar & BOOST_SERIALIZATION_NVP(rho); + ar & BOOST_SERIALIZATION_NVP(bpttSteps); ar & BOOST_SERIALIZATION_NVP(batchSize); ar & BOOST_SERIALIZATION_NVP(batchStep); ar & BOOST_SERIALIZATION_NVP(forwardStep); diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index 800eac0d58..8f17f2ecf2 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -307,6 +307,9 @@ class LSTM //! Locally-stored current rho size. size_t rhoSize; + + //! Current backpropagate through time steps. + size_t bpttSteps; }; // class LSTM } // namespace ann diff --git a/src/mlpack/methods/ann/layer/lstm_impl.hpp b/src/mlpack/methods/ann/layer/lstm_impl.hpp index 9e50b205f3..5abf58be5d 100644 --- a/src/mlpack/methods/ann/layer/lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/lstm_impl.hpp @@ -36,7 +36,8 @@ LSTM::LSTM( batchSize(0), batchStep(0), gradientStepIdx(0), - rhoSize(rho) + rhoSize(rho), + bpttSteps(0) { weights.set_size(4 * outSize * inSize + 7 * outSize + 4 * outSize * outSize, 1); @@ -53,6 +54,7 @@ void LSTM::ResetCell(const size_t size) if (batchSize == 0) return; + bpttSteps = std::min(rho, rhoSize); forwardStep = 0; gradientStepIdx = 0; backwardStep = batchSize * size - 1; @@ -248,7 +250,7 @@ void LSTM::Forward( (forwardStep + batchSize) * outSize, outSize, batchSize, false, false); forwardStep += batchSize; - if ((forwardStep / batchSize) == rho) + if ((forwardStep / batchSize) == bpttSteps) { forwardStep = 0; } @@ -318,9 +320,9 @@ void LSTM::Backward( backwardStep -= batchSize; gradientStepIdx++; - if (gradientStepIdx == rho) + if (gradientStepIdx == bpttSteps) { - backwardStep = rho - 1; + backwardStep = bpttSteps - 1; gradientStepIdx = 0; } } @@ -414,7 +416,7 @@ void LSTM::Gradient( if (gradientStep == 0) { - gradientStep = batchSize * rho - 1; + gradientStep = batchSize * bpttSteps - 1; } else { @@ -431,6 +433,7 @@ void LSTM::serialize( ar & BOOST_SERIALIZATION_NVP(inSize); ar & BOOST_SERIALIZATION_NVP(outSize); ar & BOOST_SERIALIZATION_NVP(rho); + ar & BOOST_SERIALIZATION_NVP(bpttSteps); ar & BOOST_SERIALIZATION_NVP(batchSize); ar & BOOST_SERIALIZATION_NVP(batchStep); ar & BOOST_SERIALIZATION_NVP(forwardStep); diff --git a/src/mlpack/methods/ann/rnn.hpp b/src/mlpack/methods/ann/rnn.hpp index a10d4a2b0b..f69c0b7c46 100644 --- a/src/mlpack/methods/ann/rnn.hpp +++ b/src/mlpack/methods/ann/rnn.hpp @@ -207,6 +207,11 @@ class RNN //! Modify the maximum length of backpropagation through time. size_t& Rho() { return rho; } + /** + * Reset the module infomration (weights/parameters). + */ + void ResetParameters(); + //! Serialize the model. template void Serialize(Archive& ar, const unsigned int /* version */); @@ -245,11 +250,6 @@ class RNN */ void SinglePredict(const arma::mat& predictors, arma::mat& results); - /** - * Reset the module infomration (weights/parameters). - */ - void ResetParameters(); - /** * Reset the module status by setting the current deterministic parameter * for all modules that implement the Deterministic function. diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 70acc5d915..019aca22db 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -736,6 +736,43 @@ BOOST_AUTO_TEST_CASE(SimpleAddMergeLayerTest) } } +/** + * Test the LSTM layer with a user defined rho parameter and without. + */ +BOOST_AUTO_TEST_CASE(LSTMRrhoTest) +{ + const size_t rho = 5; + arma::mat input = arma::randu(5, 1); + arma::mat target = arma::mat("1; 1; 1; 1; 1"); + RandomInitialization init(0.5, 0.5); + + // Create model with user defined rho parameter. + RNN, RandomInitialization> modelA( + input, target, rho, false, NegativeLogLikelihood<>(), init); + modelA.Add >(); + modelA.Add >(1, 10); + + // Use LSTM layer with rho. + modelA.Add >(10, 3, rho); + modelA.Add >(); + + // Create model without user defined rho parameter. + RNN > modelB( + input, target, rho, false, NegativeLogLikelihood<>(), init); + modelB.Add >(); + modelB.Add >(1, 10); + + // Use LSTM layer with rho = MAXSIZE. + modelB.Add >(10, 3); + modelB.Add >(); + + optimization::StandardSGD opt(0.1, 5, -100, false); + modelA.Train(input, target, opt); + modelB.Train(input, target, opt); + + CheckMatrices(modelB.Parameters(), modelA.Parameters()); +} + /** * LSTM layer numerically gradient test. */ @@ -779,6 +816,43 @@ BOOST_AUTO_TEST_CASE(GradientLSTMLayerTest) BOOST_REQUIRE_LE(CheckGradient(function), 1e-4); } +/** + * Test the FastLSTM layer with a user defined rho parameter and without. + */ +BOOST_AUTO_TEST_CASE(FastLSTMRrhoTest) +{ + const size_t rho = 5; + arma::mat input = arma::randu(5, 1); + arma::mat target = arma::mat("1; 1; 1; 1; 1"); + RandomInitialization init(0.5, 0.5); + + // Create model with user defined rho parameter. + RNN, RandomInitialization> modelA( + input, target, rho, false, NegativeLogLikelihood<>(), init); + modelA.Add >(); + modelA.Add >(1, 10); + + // Use FastLSTM layer with rho. + modelA.Add >(10, 3, rho); + modelA.Add >(); + + // Create model without user defined rho parameter. + RNN > modelB( + input, target, rho, false, NegativeLogLikelihood<>(), init); + modelB.Add >(); + modelB.Add >(1, 10); + + // Use FastLSTM layer with rho = MAXSIZE. + modelB.Add >(10, 3); + modelB.Add >(); + + optimization::StandardSGD opt(0.1, 5, -100, false); + modelA.Train(input, target, opt); + modelB.Train(input, target, opt); + + CheckMatrices(modelB.Parameters(), modelA.Parameters()); +} + /** * FastLSTM layer numerically gradient test. */ From 43c101727802dc4b399c9f63ef50943d0efc1f6a Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 6 Nov 2017 15:39:43 +0100 Subject: [PATCH 09/11] Refactor test to use batch size. --- src/mlpack/tests/ann_layer_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index b9353ad232..6456902ab2 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -882,8 +882,8 @@ BOOST_AUTO_TEST_CASE(GradientFastLSTMLayerTest) double Gradient(arma::mat& gradient) const { arma::mat output; - double error = model->Evaluate(model->Parameters(), 0); - model->Gradient(model->Parameters(), 0, gradient); + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); return error; } From 71bcde9be6a955d4096ad5ae1f2feb4cfa604bde Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 7 Nov 2017 00:49:28 +0100 Subject: [PATCH 10/11] Use the correct batch size for the LSTM/FastLSTM layer test. --- src/mlpack/tests/ann_layer_test.cpp | 4 ++-- src/mlpack/tests/recurrent_network_test.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 6456902ab2..9c803a3416 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -766,7 +766,7 @@ BOOST_AUTO_TEST_CASE(LSTMRrhoTest) modelB.Add >(10, 3); modelB.Add >(); - optimization::StandardSGD opt(0.1, 5, -100, false); + optimization::StandardSGD opt(0.1, 1, 5, -100, false); modelA.Train(input, target, opt); modelB.Train(input, target, opt); @@ -846,7 +846,7 @@ BOOST_AUTO_TEST_CASE(FastLSTMRrhoTest) modelB.Add >(10, 3); modelB.Add >(); - optimization::StandardSGD opt(0.1, 5, -100, false); + optimization::StandardSGD opt(0.1, 1, 5, -100, false); modelA.Train(input, target, opt); modelB.Train(input, target, opt); diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp index 5fe9182b57..38857ef9ef 100644 --- a/src/mlpack/tests/recurrent_network_test.cpp +++ b/src/mlpack/tests/recurrent_network_test.cpp @@ -455,7 +455,7 @@ void ReberGrammarTestNetwork(const size_t hiddenSize = 4, model.Add >(hiddenSize, outputSize); model.Add >(); MomentumSGD opt(0.06, 50, 2, -50000); - + arma::mat inputTemp, labelsTemp; for (size_t i = 0; i < (iterations + offset); i++) { From 853ca32b7bc008df46c00b63a2ee17e90b6b56f7 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Thu, 9 Nov 2017 21:42:21 +0100 Subject: [PATCH 11/11] Add reference to the LSTM/Fast LSTM layer and fix doxygen format. --- src/mlpack/methods/ann/layer/fast_lstm.hpp | 12 +++++++----- src/mlpack/methods/ann/layer/lstm.hpp | 13 ++++++++----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/mlpack/methods/ann/layer/fast_lstm.hpp b/src/mlpack/methods/ann/layer/fast_lstm.hpp index 3ded18274a..e853bb16d2 100644 --- a/src/mlpack/methods/ann/layer/fast_lstm.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm.hpp @@ -25,11 +25,11 @@ namespace ann /** Artificial Neural Network. */ { * and hidden state in a single step. The standard formula changes as follows: * * @f{eqnarray}{ - * i &=& sigmoid(W \cdot x + W \cdot h + b) - * f &=& sigmoid(W \cdot x + W \cdot h + b) - * z &=& tanh(W \cdot x + W \cdot h + b) - * c &=& f \cdot c + i \cdot z - * o &=& sigmoid(W \cdot x + W \cdot h + b) + * i &=& sigmoid(W \cdot x + W \cdot h + b) \\ + * f &=& sigmoid(W \cdot x + W \cdot h + b) \\ + * z &=& tanh(W \cdot x + W \cdot h + b) \\ + * c &=& f \cdot c + i \cdot z \\ + * o &=& sigmoid(W \cdot x + W \cdot h + b) \\ * h &=& o \cdot tanh(c) * @f} * @@ -47,6 +47,8 @@ namespace ann /** Artificial Neural Network. */ { * } * @endcode * + * \see LSTM for a standard implementation of the LSTM layer. + * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index 8f17f2ecf2..cd0f867136 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -28,11 +28,11 @@ namespace ann /** Artificial Neural Network. */ { * The implementation corresponds to the following algorithm: * * @f{eqnarray}{ - * i &=& sigmoid(W \cdot x + W \cdot h + W \cdot c + b) - * f &=& sigmoid(W \cdot x + W \cdot h + W \cdot c + b) - * z &=& tanh(W \cdot x + W \cdot h + b) - * c &=& f \cdot c + i \cdot z - * o &=& sigmoid(W \cdot x + W \cdot h + W \cdot c + b) + * i &=& sigmoid(W \cdot x + W \cdot h + W \cdot c + b) \\ + * f &=& sigmoid(W \cdot x + W \cdot h + W \cdot c + b) \\ + * z &=& tanh(W \cdot x + W \cdot h + b) \\ + * c &=& f \cdot c + i \cdot z \\ + * o &=& sigmoid(W \cdot x + W \cdot h + W \cdot c + b) \\ * h &=& o \cdot tanh(c) * @f} * @@ -48,6 +48,9 @@ namespace ann /** Artificial Neural Network. */ { * } * @endcode * + * \see FastLSTM for a faster LSTM version which combines the calculation of the + * input, forget, output gates and hidden state in a single step. + * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,