From 502a662a36b239cac7481a02832f682353452bf9 Mon Sep 17 00:00:00 2001 From: mulx10 Date: Wed, 20 Mar 2019 22:35:14 +0530 Subject: [PATCH 01/16] Added cellState as output params(#1782). --- src/mlpack/methods/ann/layer/lstm.hpp | 11 +++++++++++ src/mlpack/methods/ann/layer/lstm_impl.hpp | 15 +++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index 2b4deaeb97..544e3a414d 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -87,6 +87,17 @@ class LSTM template void Forward(InputType&& input, OutputType&& output); + /** + * Ordinary feed forward pass of a neural network, evaluating the function + * f(x) by propagating the activity forward through f. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + * @param cellState Cell state of LSTM. + */ + template + void Forward(InputType&& input, OutputType&& output, OutputType&& cellState); + /** * Ordinary feed backward pass of a neural network, calculating the function * f(x) by propagating x backwards trough f. Using the results from the feed diff --git a/src/mlpack/methods/ann/layer/lstm_impl.hpp b/src/mlpack/methods/ann/layer/lstm_impl.hpp index ea06e8e616..cab0378a08 100644 --- a/src/mlpack/methods/ann/layer/lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/lstm_impl.hpp @@ -159,10 +159,22 @@ void LSTM::Reset() offset, outSize, 1, false, false); } +// Forward when cellState is not needed template template void LSTM::Forward( InputType&& input, OutputType&& output) +{ + //! Locally-stored cellState. + OutputType cellState; + Forward(std::move(input), std::move(output), std::move(cellState)); +} + +// Forward when cellState is needed overloaded LSTM::Forward() +template +template +void LSTM::Forward( + InputType&& input, OutputType&& output, OutputType&& cellState) { // Check if the batch size changed, the number of cols is defines the input // batch size. @@ -249,6 +261,9 @@ void LSTM::Forward( output = OutputType(outParameter.memptr() + (forwardStep + batchSize) * outSize, outSize, batchSize, false, false); + cellState = OutputType(cell.memptr() + + forwardStep * outSize, outSize, batchSize, false, false); + forwardStep += batchSize; if ((forwardStep / batchSize) == bpttSteps) { From 3ef6010776958fad1d540cb5f41fb999f50213ed Mon Sep 17 00:00:00 2001 From: mulx10 Date: Thu, 28 Mar 2019 23:32:50 +0530 Subject: [PATCH 02/16] Added test for overloaded Forward() (#1782). --- src/mlpack/tests/ann_layer_test.cpp | 68 +++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 9aab580833..023d0c0a6d 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -901,6 +901,74 @@ BOOST_AUTO_TEST_CASE(GradientFastLSTMLayerTest) BOOST_REQUIRE_LE(CheckGradient(function), 0.2); } +/** + * Testting the overloaded Forward of LSTM Layer, + * besides ouput from the LSTM the overloaded function + * provides acces to cell state of LSTM. + */ +BOOST_AUTO_TEST_CASE(CellStateParamLSTMLayerTest) +{ + const size_t rho = 5, inputSize = 3, outputSize = 2; + + // Provide input of all ones. + arma::cube input = arma::ones(inputSize, outputSize, rho); + + arma::mat inputGate, forgetGate, outputGate, hidden; + arma::mat out_lstm, cell_lstm; + + // LSTM layer. + LSTM<> lstm(inputSize,outputSize,rho); + lstm.Reset(); + lstm.ResetCell(rho); + + // Initialize the weights to all ones. + lstm.Parameters().ones(); + + arma::mat inputWeight = arma::ones(outputSize, inputSize); + arma::mat outputWeight = arma::ones(outputSize, outputSize); + arma::mat bias = arma::ones(outputSize, input.n_cols); + arma::mat cell_calc = arma::zeros(outputSize, input.n_cols); + arma::mat out_calc = arma::zeros(outputSize, input.n_cols); + + for (size_t seqNum = 0; seqNum < rho; ++seqNum) + { + // Wrap a matrix around our data to avoid a copy. + arma::mat stepData(input.slice(seqNum).memptr(), + input.n_rows, input.n_cols, false, true); + + // Apply Forward on LSTM layer. + lstm.Forward(std::move(stepData), + std::move(out_lstm), + std::move(cell_lstm)); + + // Compute the value of cell state and output. + // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b) + inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + + outputWeight * out_calc + outputWeight % cell_calc + bias))); + + // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b) + forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + + outputWeight * out_calc + outputWeight % cell_calc + bias))); + + // z = tanh(W.dot(x) + W.dot(h) + b) + hidden = arma::tanh(inputWeight * stepData + + outputWeight * out_calc + bias); + + // c = f * c + i * z + cell_calc = forgetGate % cell_calc + inputGate % hidden; + + // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b) + outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + + outputWeight * out_calc + outputWeight % cell_calc + bias))); + + // h = o * tanh(c) + out_calc = outputGate % arma::tanh(cell_calc); + + CheckMatrices(out_lstm, out_calc, 1e-12); + CheckMatrices(cell_lstm, cell_calc, 1e-12); + } +} + /** * Check if the gradients computed by GRU cell are close enough to the * approximation of the gradients. From 705693e82e76e9ab66062e8f3bd8224f7802bf6c Mon Sep 17 00:00:00 2001 From: mulx10 Date: Fri, 29 Mar 2019 00:12:14 +0530 Subject: [PATCH 03/16] Style checks --- src/mlpack/tests/ann_layer_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 023d0c0a6d..930a8af9da 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -917,7 +917,7 @@ BOOST_AUTO_TEST_CASE(CellStateParamLSTMLayerTest) arma::mat out_lstm, cell_lstm; // LSTM layer. - LSTM<> lstm(inputSize,outputSize,rho); + LSTM<> lstm(inputSize, outputSize, rho); lstm.Reset(); lstm.ResetCell(rho); From 7a4fd56de1e0eac80aa56a866f7dc4ae300d5e41 Mon Sep 17 00:00:00 2001 From: Mehul Kumar Nirala Date: Fri, 29 Mar 2019 10:34:23 +0530 Subject: [PATCH 04/16] Update ann_layer_test.cpp --- src/mlpack/tests/ann_layer_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 930a8af9da..6232db9642 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -902,9 +902,9 @@ BOOST_AUTO_TEST_CASE(GradientFastLSTMLayerTest) } /** - * Testting the overloaded Forward of LSTM Layer, - * besides ouput from the LSTM the overloaded function - * provides acces to cell state of LSTM. + * Testting the overloaded Forward of LSTM Layer. + * Besides output, the overloaded function + * provides access to cell state of LSTM. */ BOOST_AUTO_TEST_CASE(CellStateParamLSTMLayerTest) { From 496c577ee98ce5ba7e3723ee8eaa1c57e5d476fb Mon Sep 17 00:00:00 2001 From: mulx10 Date: Sat, 30 Mar 2019 09:47:34 +0530 Subject: [PATCH 05/16] correct comments --- src/mlpack/methods/ann/layer/lstm.hpp | 2 +- src/mlpack/tests/ann_layer_test.cpp | 32 +++++++++++++-------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index 544e3a414d..2f783e0108 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -93,7 +93,7 @@ class LSTM * * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. - * @param cellState Cell state of LSTM. + * @param cellState Cell state of the LSTM. */ template void Forward(InputType&& input, OutputType&& output, OutputType&& cellState); diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 930a8af9da..a09f85fa27 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -902,11 +902,11 @@ BOOST_AUTO_TEST_CASE(GradientFastLSTMLayerTest) } /** - * Testting the overloaded Forward of LSTM Layer, + * Testing the overloaded Forward of the LSTM Layer, * besides ouput from the LSTM the overloaded function - * provides acces to cell state of LSTM. + * provides acces to cell state of the LSTM layer. */ -BOOST_AUTO_TEST_CASE(CellStateParamLSTMLayerTest) +BOOST_AUTO_TEST_CASE(ReadCellStateParamLSTMLayerTest) { const size_t rho = 5, inputSize = 3, outputSize = 2; @@ -914,7 +914,7 @@ BOOST_AUTO_TEST_CASE(CellStateParamLSTMLayerTest) arma::cube input = arma::ones(inputSize, outputSize, rho); arma::mat inputGate, forgetGate, outputGate, hidden; - arma::mat out_lstm, cell_lstm; + arma::mat outLstm, cellLstm; // LSTM layer. LSTM<> lstm(inputSize, outputSize, rho); @@ -927,8 +927,8 @@ BOOST_AUTO_TEST_CASE(CellStateParamLSTMLayerTest) arma::mat inputWeight = arma::ones(outputSize, inputSize); arma::mat outputWeight = arma::ones(outputSize, outputSize); arma::mat bias = arma::ones(outputSize, input.n_cols); - arma::mat cell_calc = arma::zeros(outputSize, input.n_cols); - arma::mat out_calc = arma::zeros(outputSize, input.n_cols); + arma::mat cellCalc = arma::zeros(outputSize, input.n_cols); + arma::mat outCalc = arma::zeros(outputSize, input.n_cols); for (size_t seqNum = 0; seqNum < rho; ++seqNum) { @@ -938,34 +938,34 @@ BOOST_AUTO_TEST_CASE(CellStateParamLSTMLayerTest) // Apply Forward on LSTM layer. lstm.Forward(std::move(stepData), - std::move(out_lstm), - std::move(cell_lstm)); + std::move(outLstm), + std::move(cellLstm)); // Compute the value of cell state and output. // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b) inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + - outputWeight * out_calc + outputWeight % cell_calc + bias))); + outputWeight * outCalc + outputWeight % cellCalc + bias))); // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b) forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + - outputWeight * out_calc + outputWeight % cell_calc + bias))); + outputWeight * outCalc + outputWeight % cellCalc + bias))); // z = tanh(W.dot(x) + W.dot(h) + b) hidden = arma::tanh(inputWeight * stepData + - outputWeight * out_calc + bias); + outputWeight * outCalc + bias); // c = f * c + i * z - cell_calc = forgetGate % cell_calc + inputGate % hidden; + cellCalc = forgetGate % cellCalc + inputGate % hidden; // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b) outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + - outputWeight * out_calc + outputWeight % cell_calc + bias))); + outputWeight * outCalc + outputWeight % cellCalc + bias))); // h = o * tanh(c) - out_calc = outputGate % arma::tanh(cell_calc); + outCalc = outputGate % arma::tanh(cellCalc); - CheckMatrices(out_lstm, out_calc, 1e-12); - CheckMatrices(cell_lstm, cell_calc, 1e-12); + CheckMatrices(outLstm, outCalc, 1e-12); + CheckMatrices(cellLstm, cellCalc, 1e-12); } } From 7942ffc43c54570ff4b263c6f0d55f03352f5c63 Mon Sep 17 00:00:00 2001 From: mulx10 Date: Sat, 30 Mar 2019 11:47:17 +0530 Subject: [PATCH 06/16] Style checks --- src/mlpack/methods/ann/layer/lstm.hpp | 2 +- src/mlpack/tests/ann_layer_test.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index a82ca4b830..cc0d4ed089 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -98,7 +98,7 @@ class LSTM */ template void Forward(InputType&& input, OutputType&& output, - OutputType&& cellState, bool useCellState); + OutputType&& cellState, bool useCellState); /** * Ordinary feed backward pass of a neural network, calculating the function diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 70cb28f7bd..10f264f168 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -1024,7 +1024,7 @@ BOOST_AUTO_TEST_CASE(WriteCellStateParamLSTMLayerTest) lstm.Forward(std::move(stepData), // Input. std::move(outLstm), // Output. std::move(cellLstm), // Cell state. - true); // Write into cell State. + true); // Write into cell State. // Compute the value of cell state and output. // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b) From f28c5b20447c980c8d2de5b7142019753d9057ad Mon Sep 17 00:00:00 2001 From: mulx10 Date: Sat, 30 Mar 2019 12:39:56 +0530 Subject: [PATCH 07/16] Remove default params --- src/mlpack/methods/ann/layer/lstm_impl.hpp | 2 +- src/mlpack/tests/ann_layer_test.cpp | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/ann/layer/lstm_impl.hpp b/src/mlpack/methods/ann/layer/lstm_impl.hpp index 82dce1e57b..457002d5df 100644 --- a/src/mlpack/methods/ann/layer/lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/lstm_impl.hpp @@ -175,7 +175,7 @@ template template void LSTM::Forward( InputType&& input, OutputType&& output, - OutputType&& cellState, bool useCellState = false) + OutputType&& cellState, bool useCellState) { // Check if the batch size changed, the number of cols is defines the input // batch size. diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 10f264f168..958408fa79 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -938,9 +938,10 @@ BOOST_AUTO_TEST_CASE(ReadCellStateParamLSTMLayerTest) input.n_rows, input.n_cols, false, true); // Apply Forward on LSTM layer. - lstm.Forward(std::move(stepData), - std::move(outLstm), - std::move(cellLstm)); + lstm.Forward(std::move(stepData), // Input. + std::move(outLstm), // Output. + std::move(cellLstm), // Cell State. + false); // Don't write into cell State. // Compute the value of cell state and output. // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b) From d4f9b2dfdd099c1b040ab8d9557824eb66de0dcd Mon Sep 17 00:00:00 2001 From: mulx10 Date: Sat, 30 Mar 2019 12:42:15 +0530 Subject: [PATCH 08/16] Style checks --- src/mlpack/methods/ann/layer/lstm_impl.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mlpack/methods/ann/layer/lstm_impl.hpp b/src/mlpack/methods/ann/layer/lstm_impl.hpp index 457002d5df..af125d6b2b 100644 --- a/src/mlpack/methods/ann/layer/lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/lstm_impl.hpp @@ -186,7 +186,6 @@ void LSTM::Forward( ResetCell(rhoSize); } - inputGate.cols(forwardStep, forwardStep + batchStep) = input2GateInputWeight * input + output2GateInputWeight * outParameter.cols(forwardStep, forwardStep + batchStep); From 18825716fe4c2e6379996ec5ca2bde6aca9e5406 Mon Sep 17 00:00:00 2001 From: mulx10 Date: Sat, 30 Mar 2019 13:05:37 +0530 Subject: [PATCH 09/16] Added Empty Cell State Test --- src/mlpack/methods/ann/layer/lstm_impl.hpp | 13 ++++++++++--- src/mlpack/tests/ann_layer_test.cpp | 22 ++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/ann/layer/lstm_impl.hpp b/src/mlpack/methods/ann/layer/lstm_impl.hpp index af125d6b2b..f823e43f31 100644 --- a/src/mlpack/methods/ann/layer/lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/lstm_impl.hpp @@ -200,10 +200,17 @@ void LSTM::Forward( if (forwardStep > 0) { - if (useCellState && !cellState.is_empty()) + if (useCellState) { - cell.cols(forwardStep - batchSize, - forwardStep - batchSize + batchStep) = cellState; + if (!cellState.is_empty()) + { + cell.cols(forwardStep - batchSize, + forwardStep - batchSize + batchStep) = cellState; + } + else + { + throw std::runtime_error("Cell parameter is empty"); + } } inputGate.cols(forwardStep, forwardStep + batchStep) += arma::repmat(cell2GateInputWeight, 1, batchSize) % diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 958408fa79..1b68fb4e4f 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -1053,6 +1053,28 @@ BOOST_AUTO_TEST_CASE(WriteCellStateParamLSTMLayerTest) CheckMatrices(outLstm, outCalc, 1e-12); CheckMatrices(cellLstm, cellCalc, 1e-12); } + + // Attempting to write empty matrix into cell state. + lstm.Reset(); + lstm.ResetCell(rho); + arma::mat stepData(input.slice(0).memptr(), + input.n_rows, input.n_cols, false, true); + + lstm.Forward(std::move(stepData), // Input. + std::move(outLstm), // Output. + std::move(cellLstm), // Cell state. + true); // Write into cell State. + + for (size_t seqNum = 1; seqNum < rho; ++seqNum) + { + arma::mat empty; + // Should throw error. + BOOST_REQUIRE_THROW(lstm.Forward(std::move(stepData), // Input. + std::move(outLstm), // Output. + std::move(empty), // Cell state. + true), // Write into cell State. + std::runtime_error); + } } /** From d1ed070e59ee9757a4653711f155fef1aa4de3de Mon Sep 17 00:00:00 2001 From: mulx10 Date: Sun, 31 Mar 2019 11:11:28 +0530 Subject: [PATCH 10/16] Incorported Reviews --- src/mlpack/methods/ann/layer/lstm.hpp | 6 ++++-- src/mlpack/methods/ann/layer/lstm_impl.hpp | 9 +++++---- src/mlpack/tests/ann_layer_test.cpp | 21 ++++++++++----------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index cc0d4ed089..7e350f81d2 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -97,8 +97,10 @@ class LSTM * @param useCellState Use the cellState passed in the LSTM Cell. */ template - void Forward(InputType&& input, OutputType&& output, - OutputType&& cellState, bool useCellState); + void Forward(InputType&& input, + OutputType&& output, + OutputType&& cellState, + bool useCellState); /** * Ordinary feed backward pass of a neural network, calculating the function diff --git a/src/mlpack/methods/ann/layer/lstm_impl.hpp b/src/mlpack/methods/ann/layer/lstm_impl.hpp index f823e43f31..753e7a4bae 100644 --- a/src/mlpack/methods/ann/layer/lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/lstm_impl.hpp @@ -173,9 +173,10 @@ void LSTM::Forward( // Forward when cellState is needed overloaded LSTM::Forward() template template -void LSTM::Forward( - InputType&& input, OutputType&& output, - OutputType&& cellState, bool useCellState) +void LSTM::Forward(InputType&& input, + OutputType&& output, + OutputType&& cellState, + bool useCellState) { // Check if the batch size changed, the number of cols is defines the input // batch size. @@ -209,7 +210,7 @@ void LSTM::Forward( } else { - throw std::runtime_error("Cell parameter is empty"); + throw std::runtime_error("Cell parameter is empty."); } } inputGate.cols(forwardStep, forwardStep + batchStep) += diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 1b68fb4e4f..1aee9e7016 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -944,26 +944,26 @@ BOOST_AUTO_TEST_CASE(ReadCellStateParamLSTMLayerTest) false); // Don't write into cell State. // Compute the value of cell state and output. - // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b) + // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + outputWeight * outCalc + outputWeight % cellCalc + bias))); - // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b) + // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + outputWeight * outCalc + outputWeight % cellCalc + bias))); - // z = tanh(W.dot(x) + W.dot(h) + b) + // z = tanh(W.dot(x) + W.dot(h) + b). hidden = arma::tanh(inputWeight * stepData + outputWeight * outCalc + bias); // c = f * c + i * z cellCalc = forgetGate % cellCalc + inputGate % hidden; - // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b) + // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + outputWeight * outCalc + outputWeight % cellCalc + bias))); - // h = o * tanh(c) + // h = o * tanh(c). outCalc = outputGate % arma::tanh(cellCalc); CheckMatrices(outLstm, outCalc, 1e-12); @@ -971,7 +971,6 @@ BOOST_AUTO_TEST_CASE(ReadCellStateParamLSTMLayerTest) } } - /** * Testing the overloaded Forward of the LSTM layer. * for writing the Cell State. Besides output the @@ -1028,26 +1027,26 @@ BOOST_AUTO_TEST_CASE(WriteCellStateParamLSTMLayerTest) true); // Write into cell State. // Compute the value of cell state and output. - // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b) + // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + outputWeight * outCalc + outputWeight % cellCalc + bias))); - // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b) + // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + outputWeight * outCalc + outputWeight % cellCalc + bias))); - // z = tanh(W.dot(x) + W.dot(h) + b) + // z = tanh(W.dot(x) + W.dot(h) + b). hidden = arma::tanh(inputWeight * stepData + outputWeight * outCalc + bias); // c = f * c + i * z cellCalc = forgetGate % cellCalc + inputGate % hidden; - // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b) + // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + outputWeight * outCalc + outputWeight % cellCalc + bias))); - // h = o * tanh(c) + // h = o * tanh(c). outCalc = outputGate % arma::tanh(cellCalc); CheckMatrices(outLstm, outCalc, 1e-12); From c947e8df8a56248ff77d147b2d1ace539810a41d Mon Sep 17 00:00:00 2001 From: mulx10 Date: Sun, 31 Mar 2019 11:14:37 +0530 Subject: [PATCH 11/16] added default param --- src/mlpack/methods/ann/layer/lstm.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index 7e350f81d2..ed3963c6f2 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -100,7 +100,7 @@ class LSTM void Forward(InputType&& input, OutputType&& output, OutputType&& cellState, - bool useCellState); + bool useCellState = false); /** * Ordinary feed backward pass of a neural network, calculating the function From c0547b252450680d9aeb0505dea1a2e58bf65731 Mon Sep 17 00:00:00 2001 From: mulx10 Date: Sun, 31 Mar 2019 19:19:17 +0530 Subject: [PATCH 12/16] Incorported Reviews --- src/mlpack/methods/ann/layer/lstm.hpp | 4 ++-- src/mlpack/methods/ann/layer/lstm_impl.hpp | 4 ++-- src/mlpack/tests/ann_layer_test.cpp | 14 ++++++-------- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index ed3963c6f2..eef9da03d9 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -78,7 +78,7 @@ class LSTM const size_t rho = std::numeric_limits::max()); /** - * Ordinary feed forward pass of a neural network, evaluating the function + * Ordinary feed-forward pass of a neural network, evaluating the function * f(x) by propagating the activity forward through f. * * @param input Input data used for evaluating the specified function. @@ -88,7 +88,7 @@ class LSTM void Forward(InputType&& input, OutputType&& output); /** - * Ordinary feed forward pass of a neural network, evaluating the function + * Ordinary feed-forward pass of a neural network, evaluating the function * f(x) by propagating the activity forward through f. * * @param input Input data used for evaluating the specified function. diff --git a/src/mlpack/methods/ann/layer/lstm_impl.hpp b/src/mlpack/methods/ann/layer/lstm_impl.hpp index 753e7a4bae..d46ef43e00 100644 --- a/src/mlpack/methods/ann/layer/lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/lstm_impl.hpp @@ -159,7 +159,7 @@ void LSTM::Reset() offset, outSize, 1, false, false); } -// Forward when cellState is not needed +// Forward when cellState is not needed. template template void LSTM::Forward( @@ -170,7 +170,7 @@ void LSTM::Forward( Forward(std::move(input), std::move(output), std::move(cellState), false); } -// Forward when cellState is needed overloaded LSTM::Forward() +// Forward when cellState is needed overloaded LSTM::Forward(). template template void LSTM::Forward(InputType&& input, diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 1aee9e7016..fac0499b24 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -902,10 +902,9 @@ BOOST_AUTO_TEST_CASE(GradientFastLSTMLayerTest) } /** - * Testing the overloaded Forward of the LSTM layer. - * for retrieving the Cell State. Besides output - * the LSTM the overloaded function provides read - * access to cell state of the LSTM layer. + * Testing the overloaded Forward() of the LSTM layer, for retrieving the Cell + * State. Besides output the overloaded function provides read access to cell + * state of the LSTM layer. */ BOOST_AUTO_TEST_CASE(ReadCellStateParamLSTMLayerTest) { @@ -972,10 +971,9 @@ BOOST_AUTO_TEST_CASE(ReadCellStateParamLSTMLayerTest) } /** - * Testing the overloaded Forward of the LSTM layer. - * for writing the Cell State. Besides output the - * the LSTM the overloaded function provides write - * access to cell state of the LSTM layer. + * Testing the overloaded Forward() of the LSTM layer, for retrieving the Cell + * State. Besides output the overloaded function provides write access to cell + * state of the LSTM layer. */ BOOST_AUTO_TEST_CASE(WriteCellStateParamLSTMLayerTest) { From 52b18b95e16218138dce56d3cd69a687b8d80e0f Mon Sep 17 00:00:00 2001 From: mulx10 Date: Sun, 31 Mar 2019 19:41:46 +0530 Subject: [PATCH 13/16] Incorported Reviews --- src/mlpack/tests/ann_layer_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index fac0499b24..83362ba95a 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -903,7 +903,7 @@ BOOST_AUTO_TEST_CASE(GradientFastLSTMLayerTest) /** * Testing the overloaded Forward() of the LSTM layer, for retrieving the Cell - * State. Besides output the overloaded function provides read access to cell + * State. Besides output, the overloaded function provides read access to cell * state of the LSTM layer. */ BOOST_AUTO_TEST_CASE(ReadCellStateParamLSTMLayerTest) @@ -972,7 +972,7 @@ BOOST_AUTO_TEST_CASE(ReadCellStateParamLSTMLayerTest) /** * Testing the overloaded Forward() of the LSTM layer, for retrieving the Cell - * State. Besides output the overloaded function provides write access to cell + * State. Besides output, the overloaded function provides write access to cell * state of the LSTM layer. */ BOOST_AUTO_TEST_CASE(WriteCellStateParamLSTMLayerTest) From 8e9e66ecc64eda15b60627fb93cc76f5c4fdf3b9 Mon Sep 17 00:00:00 2001 From: Mehul Kumar Nirala Date: Mon, 1 Apr 2019 00:31:24 +0530 Subject: [PATCH 14/16] Update ann_layer_test.cpp --- src/mlpack/tests/ann_layer_test.cpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 83362ba95a..34c130592e 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -902,8 +902,8 @@ BOOST_AUTO_TEST_CASE(GradientFastLSTMLayerTest) } /** - * Testing the overloaded Forward() of the LSTM layer, for retrieving the Cell - * State. Besides output, the overloaded function provides read access to cell + * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell + * state. Besides output, the overloaded function provides read access to cell * state of the LSTM layer. */ BOOST_AUTO_TEST_CASE(ReadCellStateParamLSTMLayerTest) @@ -936,11 +936,11 @@ BOOST_AUTO_TEST_CASE(ReadCellStateParamLSTMLayerTest) arma::mat stepData(input.slice(seqNum).memptr(), input.n_rows, input.n_cols, false, true); - // Apply Forward on LSTM layer. + // Apply Forward() on LSTM layer. lstm.Forward(std::move(stepData), // Input. std::move(outLstm), // Output. - std::move(cellLstm), // Cell State. - false); // Don't write into cell State. + std::move(cellLstm), // Cell state. + false); // Don't write into the cell state. // Compute the value of cell state and output. // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). @@ -955,7 +955,7 @@ BOOST_AUTO_TEST_CASE(ReadCellStateParamLSTMLayerTest) hidden = arma::tanh(inputWeight * stepData + outputWeight * outCalc + bias); - // c = f * c + i * z + // c = f * c + i * z. cellCalc = forgetGate % cellCalc + inputGate % hidden; // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). @@ -971,8 +971,8 @@ BOOST_AUTO_TEST_CASE(ReadCellStateParamLSTMLayerTest) } /** - * Testing the overloaded Forward() of the LSTM layer, for retrieving the Cell - * State. Besides output, the overloaded function provides write access to cell + * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell + * state. Besides output, the overloaded function provides write access to cell * state of the LSTM layer. */ BOOST_AUTO_TEST_CASE(WriteCellStateParamLSTMLayerTest) @@ -1018,11 +1018,11 @@ BOOST_AUTO_TEST_CASE(WriteCellStateParamLSTMLayerTest) cellCalc = arma::zeros(cellCalc.n_rows, cellCalc.n_cols); } - // Apply Forward on LSTM layer. + // Apply Forward() on the LSTM layer. lstm.Forward(std::move(stepData), // Input. std::move(outLstm), // Output. std::move(cellLstm), // Cell state. - true); // Write into cell State. + true); // Write into cell state. // Compute the value of cell state and output. // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). @@ -1037,7 +1037,7 @@ BOOST_AUTO_TEST_CASE(WriteCellStateParamLSTMLayerTest) hidden = arma::tanh(inputWeight * stepData + outputWeight * outCalc + bias); - // c = f * c + i * z + // c = f * c + i * z. cellCalc = forgetGate % cellCalc + inputGate % hidden; // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). @@ -1060,7 +1060,7 @@ BOOST_AUTO_TEST_CASE(WriteCellStateParamLSTMLayerTest) lstm.Forward(std::move(stepData), // Input. std::move(outLstm), // Output. std::move(cellLstm), // Cell state. - true); // Write into cell State. + true); // Write into cell state. for (size_t seqNum = 1; seqNum < rho; ++seqNum) { @@ -1069,7 +1069,7 @@ BOOST_AUTO_TEST_CASE(WriteCellStateParamLSTMLayerTest) BOOST_REQUIRE_THROW(lstm.Forward(std::move(stepData), // Input. std::move(outLstm), // Output. std::move(empty), // Cell state. - true), // Write into cell State. + true), // Write into cell state. std::runtime_error); } } From ee31f61582885e3ca83fb0926c22395448d32599 Mon Sep 17 00:00:00 2001 From: mulx10 Date: Tue, 2 Apr 2019 19:55:14 +0530 Subject: [PATCH 15/16] Documentation fix LSTM --- src/mlpack/methods/ann/layer/lstm.hpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index eef9da03d9..150df4ce2b 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -19,12 +19,7 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * An implementation of a lstm network layer. - * - * This class allows specification of the type of the activation functions used - * for the gates and cells and also of the type of the function used to - * initialize and update the peephole weights. - + * Implementation of the LSTM module class. * The implementation corresponds to the following algorithm: * * @f{eqnarray}{ From 03f7495528ca4b661cc55a974835cdd0ccc04d56 Mon Sep 17 00:00:00 2001 From: Mehul Kumar Nirala Date: Sat, 6 Apr 2019 07:54:16 +0530 Subject: [PATCH 16/16] Update lstm.hpp --- src/mlpack/methods/ann/layer/lstm.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index 150df4ce2b..0083aa0af7 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -89,7 +89,7 @@ class LSTM * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. * @param cellState Cell state of the LSTM. - * @param useCellState Use the cellState passed in the LSTM Cell. + * @param useCellState Use the cellState passed in the LSTM cell. */ template void Forward(InputType&& input,