From 89d598cd79ea90551d3f87f319b265d88a13420e Mon Sep 17 00:00:00 2001 From: Omar Shrit Date: Sat, 3 Oct 2020 16:55:40 +0200 Subject: [PATCH] Split correctly the rnn tests Signed-off-by: Omar Shrit --- src/mlpack/tests/CMakeLists.txt | 1 + src/mlpack/tests/recurrent_network_test.cpp | 956 ++++---------------- src/mlpack/tests/rnn_reber_test.cpp | 633 +++++++++++++ 3 files changed, 810 insertions(+), 780 deletions(-) create mode 100644 src/mlpack/tests/rnn_reber_test.cpp diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index e89d9ddba3..a1b840f3be 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -146,6 +146,7 @@ add_executable(mlpack_catch_test randomized_svd_test.cpp rbm_network_test.cpp recurrent_network_test.cpp + rnn_rebert_test.cpp regularized_svd_test.cpp scaling_test.cpp serialization_catch.cpp diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp index eb687190ae..f4ec8d0f1e 100644 --- a/src/mlpack/tests/recurrent_network_test.cpp +++ b/src/mlpack/tests/recurrent_network_test.cpp @@ -28,674 +28,6 @@ using namespace mlpack::ann; using namespace ens; using namespace mlpack::math; -/** - * Construct a 2-class dataset out of noisy sines. - * - * @param data Input data used to store the noisy sines. - * @param labels Labels used to store the target class of the noisy sines. - * @param points Number of points/features in a single sequence. - * @param sequences Number of sequences for each class. - * @param noise The noise factor that influences the sines. - */ -void GenerateNoisySines(arma::cube& data, - arma::mat& labels, - const size_t points, - const size_t sequences, - const double noise = 0.3) -{ - arma::colvec x = arma::linspace(0, points - 1, points) / - points * 20.0; - arma::colvec y1 = arma::sin(x + arma::as_scalar(arma::randu(1)) * 3.0); - arma::colvec y2 = arma::sin(x / 2.0 + arma::as_scalar(arma::randu(1)) * 3.0); - - data = arma::zeros(1 /* single dimension */, sequences * 2, points); - labels = arma::zeros(2 /* 2 classes */, sequences * 2); - - for (size_t seq = 0; seq < sequences; seq++) - { - arma::vec sequence = arma::randu(points) * noise + y1 + - arma::as_scalar(arma::randu(1) - 0.5) * noise; - for (size_t i = 0; i < points; ++i) - data(0, seq, i) = sequence[i]; - - labels(0, seq) = 1; - - sequence = arma::randu(points) * noise + y2 + - arma::as_scalar(arma::randu(1) - 0.5) * noise; - for (size_t i = 0; i < points; ++i) - data(0, sequences + seq, i) = sequence[i]; - - labels(1, sequences + seq) = 1; - } -} - -/** - * Train the BRNN on a larger dataset. - */ -TEST_CASE("SequenceClassificationBRNNTest", "[RecurrentNetworkTest]") -{ - // Using same test for RNN below. - size_t successes = 0; - const size_t rho = 10; - - for (size_t trial = 0; trial < 6; ++trial) - { - // Generate 12 (2 * 6) noisy sines. A single sine contains rho - // points/features. - arma::cube input; - arma::mat labelsTemp; - GenerateNoisySines(input, labelsTemp, rho, 6); - - arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); - for (size_t i = 0; i < labelsTemp.n_cols; ++i) - { - const int value = arma::as_scalar(arma::find( - arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; - labels.tube(0, i).fill(value); - } - - Add<> add(4); - Linear<> lookup(1, 4); - SigmoidLayer<> sigmoidLayer; - Linear<> linear(4, 4); - Recurrent<>* recurrent = new Recurrent<>( - add, lookup, linear, sigmoidLayer, rho); - - BRNN<> model(rho); - model.Add >(); - model.Add(recurrent); - model.Add >(4, 5); - - StandardSGD opt(0.1, 1, 500 * input.n_cols, -100); - model.Train(input, labels, opt); - INFO("Training over"); - arma::cube prediction; - model.Predict(input, prediction); - INFO("Prediction over"); - - size_t error = 0; - for (size_t i = 0; i < prediction.n_cols; ++i) - { - const int predictionValue = arma::as_scalar(arma::find( - arma::max(prediction.slice(rho - 1).col(i)) == - prediction.slice(rho - 1).col(i), 1) + 1); - - const int targetValue = arma::as_scalar(arma::find( - arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; - - if (predictionValue == targetValue) - { - error++; - } - } - - double classificationError = 1 - double(error) / prediction.n_cols; - INFO(classificationError); - if (classificationError <= 0.2) - { - ++successes; - break; - } - } - - REQUIRE(successes >= 1); -} - -/** - * Train the vanilla network on a larger dataset. - */ -TEST_CASE("SequenceClassificationTest", "[RecurrentNetworkTest]") -{ - // It isn't guaranteed that the recurrent network will converge in the - // specified number of iterations using random weights. If this works 1 of 6 - // times, I'm fine with that. All I want to know is that the network is able - // to escape from local minima and to solve the task. - size_t successes = 0; - const size_t rho = 10; - - for (size_t trial = 0; trial < 6; ++trial) - { - // Generate 12 (2 * 6) noisy sines. A single sine contains rho - // points/features. - arma::cube input; - arma::mat labelsTemp; - GenerateNoisySines(input, labelsTemp, rho, 6); - - arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); - for (size_t i = 0; i < labelsTemp.n_cols; ++i) - { - const int value = arma::as_scalar(arma::find( - arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; - labels.tube(0, i).fill(value); - } - - /** - * Construct a network with 1 input unit, 4 hidden units and 10 output - * units. The hidden layer is connected to itself. The network structure - * looks like: - * - * Input Hidden Output - * Layer(1) Layer(4) Layer(10) - * +-----+ +-----+ +-----+ - * | | | | | | - * | +------>| +------>| | - * | | ..>| | | | - * +-----+ . +--+--+ +-----+ - * . . - * . . - * ....... - */ - Add<> add(4); - Linear<> lookup(1, 4); - SigmoidLayer<> sigmoidLayer; - Linear<> linear(4, 4); - Recurrent<>* recurrent = new Recurrent<>( - add, lookup, linear, sigmoidLayer, rho); - - RNN<> model(rho); - model.Add >(); - model.Add(recurrent); - model.Add >(4, 10); - model.Add >(); - - StandardSGD opt(0.1, 1, 500 * input.n_cols, -100); - model.Train(input, labels, opt); - - arma::cube prediction; - model.Predict(input, prediction); - - size_t error = 0; - for (size_t i = 0; i < prediction.n_cols; ++i) - { - const int predictionValue = arma::as_scalar(arma::find( - arma::max(prediction.slice(rho - 1).col(i)) == - prediction.slice(rho - 1).col(i), 1) + 1); - - const int targetValue = arma::as_scalar(arma::find( - arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; - - if (predictionValue == targetValue) - { - error++; - } - } - - double classificationError = 1 - double(error) / prediction.n_cols; - if (classificationError <= 0.2) - { - ++successes; - break; - } - } - - REQUIRE(successes >= 1); -} - -/** - * Generate a random Reber grammar. - * - * For more information, see the following thesis. - * - * @code - * @misc{Gers2001, - * author = {Felix Gers}, - * title = {Long Short-Term Memory in Recurrent Neural Networks}, - * year = {2001} - * } - * @endcode - * - * @param transitions Reber grammar transition matrix. - * @param reber The generated Reber grammar string. - */ -void GenerateReber(const arma::Mat& transitions, std::string& reber) -{ - size_t idx = 0; - reber = "B"; - - do - { - const int grammerIdx = rand() % 2; - reber += arma::as_scalar(transitions.submat(idx, grammerIdx, idx, - grammerIdx)); - - idx = arma::as_scalar(transitions.submat(idx, grammerIdx + 2, idx, - grammerIdx + 2)) - '0'; - } while (idx != 0); -} - -/** - * Generate a random recursive Reber grammar. - * - * @param transitions Recursive Reber grammar transition matrix. - * @param averageRecursion Average recursive depth of the reber grammar. - * @param maxRecursion Maximum recursive depth of reber grammar. - * @param reber The generated embedded Reber grammar string. - * @param addEnd Add ending 'E' to the generated grammar. - */ -void GenerateRecursiveReber(const arma::Mat& transitions, - size_t averageRecursion, - size_t maxRecursion, - std::string& reber, - bool addEnd = true) -{ - char c = (rand() % averageRecursion) == 1 ? 'P' : 'T'; - - if (maxRecursion == 1 || c == 'T') - { - c = 'T'; - GenerateReber(transitions, reber); - } - else - { - GenerateRecursiveReber(transitions, averageRecursion, --maxRecursion, - reber, false); - } - - reber = c + reber + c; - - if (addEnd) - { - reber = "B" + reber + "E"; - } -} - -/** - * Convert a unit vector to a Reber symbol. - * - * @param translation The unit vector to be converted. - * @param symbol The converted unit vector stored as Reber symbol. - */ -template -void ReberReverseTranslation(const MatType& translation, char& symbol) -{ - arma::Col symbols; - symbols << 'B' << 'T' << 'S' << 'X' << 'P' << 'V' << 'E' << arma::endr; - const int idx = arma::as_scalar(arma::find(translation == 1, 1, "first")); - - symbol = symbols(idx); -} - -/** - * Convert a Reber symbol to a unit vector. - * - * @param symbol Reber symbol to be converted. - * @param translation The converted symbol stored as unit vector. - */ -void ReberTranslation(const char symbol, arma::colvec& translation) -{ - arma::Col symbols; - symbols << 'B' << 'T' << 'S' << 'X' << 'P' << 'V' << 'E' << arma::endr; - const int idx = arma::as_scalar(arma::find(symbols == symbol, 1, "first")); - - translation = arma::zeros(7); - translation(idx) = 1; -} - -/** - * Given a Reber string, return a Reber string with all reachable next symbols. - * - * @param transitions The Reber transistion matrix. - * @param reber The Reber string used to generate all reachable next symbols. - * @param nextReber All reachable next symbols. - */ -void GenerateNextReber(const arma::Mat& transitions, - const std::string& reber, std::string& nextReber) -{ - size_t idx = 0; - - for (size_t grammer = 1; grammer < reber.length(); grammer++) - { - const int grammerIdx = arma::as_scalar(arma::find( - transitions.row(idx) == reber[grammer], 1, "first")); - - idx = arma::as_scalar(transitions.submat(idx, grammerIdx + 2, idx, - grammerIdx + 2)) - '0'; - } - - nextReber = arma::as_scalar(transitions.submat(idx, 0, idx, 0)); - nextReber += arma::as_scalar(transitions.submat(idx, 1, idx, 1)); -} - -/** - * Given a recursive Reber string, return a Reber string with all - * reachable next symbols. - * - * @param transitions The Reber transistion matrix. - * @param reber The Reber string used to generate all reachable next symbols. - * @param nextReber All reachable next symbols. - */ -void GenerateNextRecursiveReber(const arma::Mat& transitions, - const std::string& reber, - std::string& nextReber) -{ - size_t state = 0; - size_t numPs = 0; - - for (size_t cIndex = 0; cIndex < reber.length(); cIndex++) - { - char c = reber[cIndex]; - - if (c == 'B' && state == 0) - { - state = 1; - } - else if (c == 'P' && state == 1) - { - numPs++; - state = 1; - } - else if (c == 'T' && state == 1) - { - state = 2; - } - else if (c == 'B' && state == 2) - { - size_t pos = reber.find('E'); - if (pos != std::string::npos) - { - cIndex = pos; - state = 4; - } - else - { - GenerateNextReber(transitions, reber.substr(cIndex), nextReber); - state = 3; - } - } - else if (c == 'T' && state == 4) - { - state = 5; - } - else if (c == 'P' && state == 5) - { - numPs--; - state = 5; - } - } - - if (state == 0 || state == 2) - { - nextReber = "B"; - } - else if (state == 1) - { - nextReber = "PT"; - } - else if (state == 4) - { - nextReber = "T"; - } - else if (state == 5) - { - if (numPs == 0) - { - nextReber = "E"; - } - else - { - nextReber = "P"; - } - } -} - -/** - * @brief Creates the reber grammar data for tests. - * - * @param trainInput The train data - * @param trainLabels The train labels - * @param testInput The test input - * @param recursive whether recursive Reber - * @param trainReberGrammarCount The number of training set - * @param testReberGrammarCount The number of test set - * @param averageRecursion Average recursion - * @param maxRecursion Max recursion - * @return arma::Mat The Reber state translation to be used. - */ -arma::Mat GenerateReberGrammarData( - arma::field& trainInput, - arma::field& trainLabels, - arma::field& testInput, - bool recursive = false, - const size_t trainReberGrammarCount = 700, - const size_t testReberGrammarCount = 250, - const size_t averageRecursion = 3, - const size_t maxRecursion = 5) -{ - // Reber state transition matrix. (The last two columns are the indices to the - // next path). - arma::Mat transitions; - transitions << 'T' << 'P' << '1' << '2' << arma::endr - << 'X' << 'S' << '3' << '1' << arma::endr - << 'V' << 'T' << '4' << '2' << arma::endr - << 'X' << 'S' << '2' << '5' << arma::endr - << 'P' << 'V' << '3' << '5' << arma::endr - << 'E' << 'E' << '0' << '0' << arma::endr; - - - std::string trainReber, testReber; - - arma::colvec translation; - - // Generate the training data. - for (size_t i = 0; i < trainReberGrammarCount; ++i) - { - if (recursive) - GenerateRecursiveReber(transitions, 3, 5, trainReber); - else - GenerateReber(transitions, trainReber); - - for (size_t j = 0; j < trainReber.length() - 1; ++j) - { - ReberTranslation(trainReber[j], translation); - trainInput(0, i) = arma::join_cols(trainInput(0, i), translation); - - ReberTranslation(trainReber[j + 1], translation); - trainLabels(0, i) = arma::join_cols(trainLabels(0, i), translation); - } - } - - // Generate the test data. - for (size_t i = 0; i < testReberGrammarCount; ++i) - { - if (recursive) - GenerateRecursiveReber(transitions, averageRecursion, maxRecursion, - testReber); - else - GenerateReber(transitions, testReber); - - for (size_t j = 0; j < testReber.length() - 1; ++j) - { - ReberTranslation(testReber[j], translation); - testInput(0, i) = arma::join_cols(testInput(0, i), translation); - } - } - - return transitions; -} - -/** - * Train the specified network and the construct a Reber grammar dataset. - */ -template -void ReberGrammarTestNetwork(ModelType& model, - const bool recursive = false, - const size_t averageRecursion = 3, - const size_t maxRecursion = 5, - const size_t iterations = 10, - const size_t trials = 5) -{ - const size_t trainReberGrammarCount = 700; - const size_t testReberGrammarCount = 250; - - arma::field trainInput(1, trainReberGrammarCount); - arma::field trainLabels(1, trainReberGrammarCount); - arma::field testInput(1, testReberGrammarCount); - - arma::Mat transitions = - GenerateReberGrammarData(trainInput, - trainLabels, - testInput, - recursive, - trainReberGrammarCount, - testReberGrammarCount, - averageRecursion, - maxRecursion); - - /* - * Construct a network with 7 input units, layerSize hidden units and 7 output - * units. The hidden layer is connected to itself. The network structure looks - * like: - * - * Input Hidden Output - * Layer(7) Layer(layerSize) Layer(7) - * +-----+ +-----+ +-----+ - * | | | | | | - * | +------>| +------>| | - * | | ..>| | | | - * +-----+ . +--+--+ +-- ---+ - * . . - * . . - * ....... - */ - // It isn't guaranteed that the recurrent network will converge in the - // specified number of iterations using random weights. If this works 1 of 5 - // times, I'm fine with that. All I want to know is that the network is able - // to escape from local minima and to solve the task. - size_t successes = 0; - size_t offset = 0; - const size_t inputSize = 7; - for (size_t trial = 0; trial < trials; ++trial) - { - // Reset model before using for next trial. - model.Reset(); - MomentumSGD opt(0.06, 50, 2, -50000); - - arma::cube inputTemp, labelsTemp; - for (size_t iteration = 0; iteration < (iterations + offset); iteration++) - { - for (size_t j = 0; j < trainReberGrammarCount; ++j) - { - // Each sequence may be a different length, so we need to extract them - // manually. We will reshape them into a cube with each slice equal to - // a time step. - inputTemp = arma::cube(trainInput.at(0, j).memptr(), inputSize, 1, - trainInput.at(0, j).n_elem / inputSize, false, true); - labelsTemp = arma::cube(trainLabels.at(0, j).memptr(), inputSize, 1, - trainInput.at(0, j).n_elem / inputSize, false, true); - - model.Rho() = inputTemp.n_elem / inputSize; - model.Train(inputTemp, labelsTemp, opt); - opt.ResetPolicy() = false; - } - } - - double error = 0; - - // Ask the network to predict the next Reber grammar in the given sequence. - for (size_t i = 0; i < testReberGrammarCount; ++i) - { - arma::cube prediction; - arma::cube input(testInput.at(0, i).memptr(), inputSize, 1, - testInput.at(0, i).n_elem / inputSize, false, true); - - model.Rho() = input.n_elem / inputSize; - model.Predict(input, prediction); - - const size_t reberGrammerSize = 7; - std::string inputReber = ""; - - size_t reberError = 0; - - for (size_t j = 0; j < (prediction.n_elem / reberGrammerSize); ++j) - { - char predictedSymbol, inputSymbol; - std::string reberChoices; - - arma::umat output = (prediction.slice(j) == (arma::ones( - reberGrammerSize, 1) * - arma::as_scalar(arma::max(prediction.slice(j))))); - - ReberReverseTranslation(output, predictedSymbol); - ReberReverseTranslation(input.slice(j), inputSymbol); - inputReber += inputSymbol; - - if (recursive) - GenerateNextRecursiveReber(transitions, inputReber, reberChoices); - else - GenerateNextReber(transitions, inputReber, reberChoices); - - if (reberChoices.find(predictedSymbol) != std::string::npos) - reberError++; - } - - if (reberError != (prediction.n_elem / reberGrammerSize)) - error += 1; - } - - error /= testReberGrammarCount; - if (error <= 0.3) - { - ++successes; - break; - } - - offset += 3; - } - - REQUIRE(successes >= 1); -} - -/** - * Train the specified networks on an embedded Reber grammar dataset. - */ -TEST_CASE("LSTMReberGrammarTest", "[RecurrentNetworkTest]") -{ - RNN > model(5); - model.Add >(7, 10); - model.Add >(10, 10); - model.Add >(10, 7); - model.Add >(); - ReberGrammarTestNetwork(model, false); -} - -/** - * Train the specified networks on an embedded Reber grammar dataset. - */ -TEST_CASE("FastLSTMReberGrammarTest", "[RecurrentNetworkTest]") -{ - RNN > model(5); - model.Add >(7, 8); - model.Add >(8, 8); - model.Add >(8, 7); - model.Add >(); - ReberGrammarTestNetwork(model, false); -} - -/** - * Train the specified networks on an embedded Reber grammar dataset. - */ -TEST_CASE("GRURecursiveReberGrammarTest", "[RecurrentNetworkTest]") -{ - RNN > model(5); - model.Add >(7, 16); - model.Add >(16, 16); - model.Add >(16, 7); - model.Add >(); - ReberGrammarTestNetwork(model, true, 3, 5, 10, 7); -} - -/** - * Train BLSTM on an embedded Reber grammar dataset. - */ -TEST_CASE("BRNNReberGrammarTest", "[RecurrentNetworkTest]") -{ - BRNN, AddMerge<>, SigmoidLayer<> > model(5); - model.Add >(7, 10); - model.Add >(10, 10); - model.Add >(10, 7); - ReberGrammarTestNetwork(model, false, 3, 5, 1); -} - /* * This sample is a simplified version of Derek D. Monner's Distracted Sequence * Recall task, which involves 10 symbols: @@ -1042,142 +374,206 @@ TEST_CASE("RNNSerializationTest", "[RecurrentNetworkTest]") } /** - * Test RNN with a custom layer. + * Construct a 2-class dataset out of noisy sines. + * + * @param data Input data used to store the noisy sines. + * @param labels Labels used to store the target class of the noisy sines. + * @param points Number of points/features in a single sequence. + * @param sequences Number of sequences for each class. + * @param noise The noise factor that influences the sines. */ -void ReberGrammarTestCustomNetwork(const size_t hiddenSize = 4, - const bool recursive = false, - const size_t iterations = 10) +void GenerateNoisySines(arma::cube& data, + arma::mat& labels, + const size_t points, + const size_t sequences, + const double noise = 0.3) { - const size_t trainReberGrammarCount = 700; - const size_t testReberGrammarCount = 250; + arma::colvec x = arma::linspace(0, points - 1, points) / + points * 20.0; + arma::colvec y1 = arma::sin(x + arma::as_scalar(arma::randu(1)) * 3.0); + arma::colvec y2 = arma::sin(x / 2.0 + arma::as_scalar(arma::randu(1)) * 3.0); - arma::field trainInput(1, trainReberGrammarCount); - arma::field trainLabels(1, trainReberGrammarCount); - arma::field testInput(1, testReberGrammarCount); + data = arma::zeros(1 /* single dimension */, sequences * 2, points); + labels = arma::zeros(2 /* 2 classes */, sequences * 2); - arma::Mat transitions = - GenerateReberGrammarData(trainInput, - trainLabels, - testInput, - recursive, - trainReberGrammarCount, - testReberGrammarCount); - - /* - * Construct a network with 7 input units, layerSize hidden units and 7 output - * units. The hidden layer is connected to itself. The network structure looks - * like: - * - * Input Hidden Output - * Layer(7) Layer(layerSize) Layer(7) - * +-----+ +-----+ +-----+ - * | | | | | | - * | +------>| +------>| | - * | | ..>| | | | - * +-----+ . +--+--+ +-- ---+ - * . . - * . . - * ....... - */ - // It isn't guaranteed that the recurrent network will converge in the - // specified number of iterations using random weights. If this works 1 of 10 - // times, I'm fine with that. All I want to know is that the network is able - // to escape from local minima and to solve the task. - size_t successes = 0; - size_t offset = 0; - for (size_t trial = 0; trial < 10; ++trial) + for (size_t seq = 0; seq < sequences; seq++) { - const size_t outputSize = 7; - const size_t inputSize = 7; + arma::vec sequence = arma::randu(points) * noise + y1 + + arma::as_scalar(arma::randu(1) - 0.5) * noise; + for (size_t i = 0; i < points; ++i) + data(0, seq, i) = sequence[i]; - RNN, RandomInitialization, CustomLayer<> > model(5); - model.Add >(inputSize, hiddenSize); - model.Add >(hiddenSize, hiddenSize); - model.Add >(hiddenSize, outputSize); - model.Add >(); - MomentumSGD opt(0.06, 50, 2, -50000); + labels(0, seq) = 1; - arma::cube inputTemp, labelsTemp; - for (size_t iteration = 0; iteration < (iterations + offset); iteration++) + sequence = arma::randu(points) * noise + y2 + + arma::as_scalar(arma::randu(1) - 0.5) * noise; + for (size_t i = 0; i < points; ++i) + data(0, sequences + seq, i) = sequence[i]; + + labels(1, sequences + seq) = 1; + } +} + +/** + * Train the BRNN on a larger dataset. + */ +TEST_CASE("SequenceClassificationBRNNTest", "[RecurrentNetworkTest]") +{ + // Using same test for RNN below. + size_t successes = 0; + const size_t rho = 10; + + for (size_t trial = 0; trial < 6; ++trial) + { + // Generate 12 (2 * 6) noisy sines. A single sine contains rho + // points/features. + arma::cube input; + arma::mat labelsTemp; + GenerateNoisySines(input, labelsTemp, rho, 6); + + arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); + for (size_t i = 0; i < labelsTemp.n_cols; ++i) { - for (size_t j = 0; j < trainReberGrammarCount; ++j) - { - // Each sequence may be a different length, so we need to extract them - // manually. We will reshape them into a cube with each slice equal to - // a time step. - inputTemp = arma::cube(trainInput.at(0, j).memptr(), inputSize, 1, - trainInput.at(0, j).n_elem / inputSize, false, true); - labelsTemp = arma::cube(trainLabels.at(0, j).memptr(), inputSize, 1, - trainInput.at(0, j).n_elem / inputSize, false, true); + const int value = arma::as_scalar(arma::find( + arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; + labels.tube(0, i).fill(value); + } - model.Rho() = inputTemp.n_elem / inputSize; - model.Train(inputTemp, labelsTemp, opt); - opt.ResetPolicy() = false; + Add<> add(4); + Linear<> lookup(1, 4); + SigmoidLayer<> sigmoidLayer; + Linear<> linear(4, 4); + Recurrent<>* recurrent = new Recurrent<>( + add, lookup, linear, sigmoidLayer, rho); + + BRNN<> model(rho); + model.Add >(); + model.Add(recurrent); + model.Add >(4, 5); + + StandardSGD opt(0.1, 1, 500 * input.n_cols, -100); + model.Train(input, labels, opt); + INFO("Training over"); + arma::cube prediction; + model.Predict(input, prediction); + INFO("Prediction over"); + + size_t error = 0; + for (size_t i = 0; i < prediction.n_cols; ++i) + { + const int predictionValue = arma::as_scalar(arma::find( + arma::max(prediction.slice(rho - 1).col(i)) == + prediction.slice(rho - 1).col(i), 1) + 1); + + const int targetValue = arma::as_scalar(arma::find( + arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; + + if (predictionValue == targetValue) + { + error++; } } - double error = 0; - - // Ask the network to predict the next Reber grammar in the given sequence. - for (size_t i = 0; i < testReberGrammarCount; ++i) - { - arma::cube prediction; - arma::cube input(testInput.at(0, i).memptr(), inputSize, 1, - testInput.at(0, i).n_elem / inputSize, false, true); - - model.Rho() = input.n_elem / inputSize; - model.Predict(input, prediction); - - const size_t reberGrammerSize = 7; - std::string inputReber = ""; - - size_t reberError = 0; - - for (size_t j = 0; j < (prediction.n_elem / reberGrammerSize); ++j) - { - char predictedSymbol, inputSymbol; - std::string reberChoices; - - arma::umat output = (prediction.slice(j) == (arma::ones( - reberGrammerSize, 1) * - arma::as_scalar(arma::max(prediction.slice(j))))); - - ReberReverseTranslation(output, predictedSymbol); - ReberReverseTranslation(input.slice(j), inputSymbol); - inputReber += inputSymbol; - - if (recursive) - GenerateNextRecursiveReber(transitions, inputReber, reberChoices); - else - GenerateNextReber(transitions, inputReber, reberChoices); - - if (reberChoices.find(predictedSymbol) != std::string::npos) - reberError++; - } - - if (reberError != (prediction.n_elem / reberGrammerSize)) - error += 1; - } - - error /= testReberGrammarCount; - if (error <= 0.35) + double classificationError = 1 - double(error) / prediction.n_cols; + INFO(classificationError); + if (classificationError <= 0.2) { ++successes; break; } - - offset += 3; } REQUIRE(successes >= 1); } /** - * Train the specified networks on an embedded Reber grammar dataset. + * Train the vanilla network on a larger dataset. */ -TEST_CASE("CustomRecursiveReberGrammarTest", "[RecurrentNetworkTest]") +TEST_CASE("SequenceClassificationTest", "[RecurrentNetworkTest]") { - ReberGrammarTestCustomNetwork(16, true); + // It isn't guaranteed that the recurrent network will converge in the + // specified number of iterations using random weights. If this works 1 of 6 + // times, I'm fine with that. All I want to know is that the network is able + // to escape from local minima and to solve the task. + size_t successes = 0; + const size_t rho = 10; + + for (size_t trial = 0; trial < 6; ++trial) + { + // Generate 12 (2 * 6) noisy sines. A single sine contains rho + // points/features. + arma::cube input; + arma::mat labelsTemp; + GenerateNoisySines(input, labelsTemp, rho, 6); + + arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); + for (size_t i = 0; i < labelsTemp.n_cols; ++i) + { + const int value = arma::as_scalar(arma::find( + arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; + labels.tube(0, i).fill(value); + } + + /** + * Construct a network with 1 input unit, 4 hidden units and 10 output + * units. The hidden layer is connected to itself. The network structure + * looks like: + * + * Input Hidden Output + * Layer(1) Layer(4) Layer(10) + * +-----+ +-----+ +-----+ + * | | | | | | + * | +------>| +------>| | + * | | ..>| | | | + * +-----+ . +--+--+ +-----+ + * . . + * . . + * ....... + */ + Add<> add(4); + Linear<> lookup(1, 4); + SigmoidLayer<> sigmoidLayer; + Linear<> linear(4, 4); + Recurrent<>* recurrent = new Recurrent<>( + add, lookup, linear, sigmoidLayer, rho); + + RNN<> model(rho); + model.Add >(); + model.Add(recurrent); + model.Add >(4, 10); + model.Add >(); + + StandardSGD opt(0.1, 1, 500 * input.n_cols, -100); + model.Train(input, labels, opt); + + arma::cube prediction; + model.Predict(input, prediction); + + size_t error = 0; + for (size_t i = 0; i < prediction.n_cols; ++i) + { + const int predictionValue = arma::as_scalar(arma::find( + arma::max(prediction.slice(rho - 1).col(i)) == + prediction.slice(rho - 1).col(i), 1) + 1); + + const int targetValue = arma::as_scalar(arma::find( + arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; + + if (predictionValue == targetValue) + { + error++; + } + } + + double classificationError = 1 - double(error) / prediction.n_cols; + if (classificationError <= 0.2) + { + ++successes; + break; + } + } + + REQUIRE(successes >= 1); } /** diff --git a/src/mlpack/tests/rnn_reber_test.cpp b/src/mlpack/tests/rnn_reber_test.cpp new file mode 100644 index 0000000000..b583a1c64a --- /dev/null +++ b/src/mlpack/tests/rnn_reber_test.cpp @@ -0,0 +1,633 @@ +/** + * @file tests/recurrent_network_test.cpp + * @author Marcus Edel + * + * Tests the recurrent network. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "catch.hpp" +#include "serialization_catch.hpp" +#include "custom_layer.hpp" + +using namespace mlpack; +using namespace mlpack::ann; +using namespace ens; +using namespace mlpack::math; + +/** + * Generate a random Reber grammar. + * + * For more information, see the following thesis. + * + * @code + * @misc{Gers2001, + * author = {Felix Gers}, + * title = {Long Short-Term Memory in Recurrent Neural Networks}, + * year = {2001} + * } + * @endcode + * + * @param transitions Reber grammar transition matrix. + * @param reber The generated Reber grammar string. + */ +void GenerateReber(const arma::Mat& transitions, std::string& reber) +{ + size_t idx = 0; + reber = "B"; + + do + { + const int grammerIdx = rand() % 2; + reber += arma::as_scalar(transitions.submat(idx, grammerIdx, idx, + grammerIdx)); + + idx = arma::as_scalar(transitions.submat(idx, grammerIdx + 2, idx, + grammerIdx + 2)) - '0'; + } while (idx != 0); +} + +/** + * Generate a random recursive Reber grammar. + * + * @param transitions Recursive Reber grammar transition matrix. + * @param averageRecursion Average recursive depth of the reber grammar. + * @param maxRecursion Maximum recursive depth of reber grammar. + * @param reber The generated embedded Reber grammar string. + * @param addEnd Add ending 'E' to the generated grammar. + */ +void GenerateRecursiveReber(const arma::Mat& transitions, + size_t averageRecursion, + size_t maxRecursion, + std::string& reber, + bool addEnd = true) +{ + char c = (rand() % averageRecursion) == 1 ? 'P' : 'T'; + + if (maxRecursion == 1 || c == 'T') + { + c = 'T'; + GenerateReber(transitions, reber); + } + else + { + GenerateRecursiveReber(transitions, averageRecursion, --maxRecursion, + reber, false); + } + + reber = c + reber + c; + + if (addEnd) + { + reber = "B" + reber + "E"; + } +} + +/** + * Convert a unit vector to a Reber symbol. + * + * @param translation The unit vector to be converted. + * @param symbol The converted unit vector stored as Reber symbol. + */ +template +void ReberReverseTranslation(const MatType& translation, char& symbol) +{ + arma::Col symbols; + symbols << 'B' << 'T' << 'S' << 'X' << 'P' << 'V' << 'E' << arma::endr; + const int idx = arma::as_scalar(arma::find(translation == 1, 1, "first")); + + symbol = symbols(idx); +} + +/** + * Convert a Reber symbol to a unit vector. + * + * @param symbol Reber symbol to be converted. + * @param translation The converted symbol stored as unit vector. + */ +void ReberTranslation(const char symbol, arma::colvec& translation) +{ + arma::Col symbols; + symbols << 'B' << 'T' << 'S' << 'X' << 'P' << 'V' << 'E' << arma::endr; + const int idx = arma::as_scalar(arma::find(symbols == symbol, 1, "first")); + + translation = arma::zeros(7); + translation(idx) = 1; +} + +/** + * Given a Reber string, return a Reber string with all reachable next symbols. + * + * @param transitions The Reber transistion matrix. + * @param reber The Reber string used to generate all reachable next symbols. + * @param nextReber All reachable next symbols. + */ +void GenerateNextReber(const arma::Mat& transitions, + const std::string& reber, std::string& nextReber) +{ + size_t idx = 0; + + for (size_t grammer = 1; grammer < reber.length(); grammer++) + { + const int grammerIdx = arma::as_scalar(arma::find( + transitions.row(idx) == reber[grammer], 1, "first")); + + idx = arma::as_scalar(transitions.submat(idx, grammerIdx + 2, idx, + grammerIdx + 2)) - '0'; + } + + nextReber = arma::as_scalar(transitions.submat(idx, 0, idx, 0)); + nextReber += arma::as_scalar(transitions.submat(idx, 1, idx, 1)); +} + +/** + * Given a recursive Reber string, return a Reber string with all + * reachable next symbols. + * + * @param transitions The Reber transistion matrix. + * @param reber The Reber string used to generate all reachable next symbols. + * @param nextReber All reachable next symbols. + */ +void GenerateNextRecursiveReber(const arma::Mat& transitions, + const std::string& reber, + std::string& nextReber) +{ + size_t state = 0; + size_t numPs = 0; + + for (size_t cIndex = 0; cIndex < reber.length(); cIndex++) + { + char c = reber[cIndex]; + + if (c == 'B' && state == 0) + { + state = 1; + } + else if (c == 'P' && state == 1) + { + numPs++; + state = 1; + } + else if (c == 'T' && state == 1) + { + state = 2; + } + else if (c == 'B' && state == 2) + { + size_t pos = reber.find('E'); + if (pos != std::string::npos) + { + cIndex = pos; + state = 4; + } + else + { + GenerateNextReber(transitions, reber.substr(cIndex), nextReber); + state = 3; + } + } + else if (c == 'T' && state == 4) + { + state = 5; + } + else if (c == 'P' && state == 5) + { + numPs--; + state = 5; + } + } + + if (state == 0 || state == 2) + { + nextReber = "B"; + } + else if (state == 1) + { + nextReber = "PT"; + } + else if (state == 4) + { + nextReber = "T"; + } + else if (state == 5) + { + if (numPs == 0) + { + nextReber = "E"; + } + else + { + nextReber = "P"; + } + } +} + +/** + * @brief Creates the reber grammar data for tests. + * + * @param trainInput The train data + * @param trainLabels The train labels + * @param testInput The test input + * @param recursive whether recursive Reber + * @param trainReberGrammarCount The number of training set + * @param testReberGrammarCount The number of test set + * @param averageRecursion Average recursion + * @param maxRecursion Max recursion + * @return arma::Mat The Reber state translation to be used. + */ +arma::Mat GenerateReberGrammarData( + arma::field& trainInput, + arma::field& trainLabels, + arma::field& testInput, + bool recursive = false, + const size_t trainReberGrammarCount = 700, + const size_t testReberGrammarCount = 250, + const size_t averageRecursion = 3, + const size_t maxRecursion = 5) +{ + // Reber state transition matrix. (The last two columns are the indices to the + // next path). + arma::Mat transitions; + transitions << 'T' << 'P' << '1' << '2' << arma::endr + << 'X' << 'S' << '3' << '1' << arma::endr + << 'V' << 'T' << '4' << '2' << arma::endr + << 'X' << 'S' << '2' << '5' << arma::endr + << 'P' << 'V' << '3' << '5' << arma::endr + << 'E' << 'E' << '0' << '0' << arma::endr; + + + std::string trainReber, testReber; + + arma::colvec translation; + + // Generate the training data. + for (size_t i = 0; i < trainReberGrammarCount; ++i) + { + if (recursive) + GenerateRecursiveReber(transitions, 3, 5, trainReber); + else + GenerateReber(transitions, trainReber); + + for (size_t j = 0; j < trainReber.length() - 1; ++j) + { + ReberTranslation(trainReber[j], translation); + trainInput(0, i) = arma::join_cols(trainInput(0, i), translation); + + ReberTranslation(trainReber[j + 1], translation); + trainLabels(0, i) = arma::join_cols(trainLabels(0, i), translation); + } + } + + // Generate the test data. + for (size_t i = 0; i < testReberGrammarCount; ++i) + { + if (recursive) + GenerateRecursiveReber(transitions, averageRecursion, maxRecursion, + testReber); + else + GenerateReber(transitions, testReber); + + for (size_t j = 0; j < testReber.length() - 1; ++j) + { + ReberTranslation(testReber[j], translation); + testInput(0, i) = arma::join_cols(testInput(0, i), translation); + } + } + + return transitions; +} + +/** + * Train the specified network and the construct a Reber grammar dataset. + */ +template +void ReberGrammarTestNetwork(ModelType& model, + const bool recursive = false, + const size_t averageRecursion = 3, + const size_t maxRecursion = 5, + const size_t iterations = 10, + const size_t trials = 5) +{ + const size_t trainReberGrammarCount = 700; + const size_t testReberGrammarCount = 250; + + arma::field trainInput(1, trainReberGrammarCount); + arma::field trainLabels(1, trainReberGrammarCount); + arma::field testInput(1, testReberGrammarCount); + + arma::Mat transitions = + GenerateReberGrammarData(trainInput, + trainLabels, + testInput, + recursive, + trainReberGrammarCount, + testReberGrammarCount, + averageRecursion, + maxRecursion); + + /* + * Construct a network with 7 input units, layerSize hidden units and 7 output + * units. The hidden layer is connected to itself. The network structure looks + * like: + * + * Input Hidden Output + * Layer(7) Layer(layerSize) Layer(7) + * +-----+ +-----+ +-----+ + * | | | | | | + * | +------>| +------>| | + * | | ..>| | | | + * +-----+ . +--+--+ +-- ---+ + * . . + * . . + * ....... + */ + // It isn't guaranteed that the recurrent network will converge in the + // specified number of iterations using random weights. If this works 1 of 5 + // times, I'm fine with that. All I want to know is that the network is able + // to escape from local minima and to solve the task. + size_t successes = 0; + size_t offset = 0; + const size_t inputSize = 7; + for (size_t trial = 0; trial < trials; ++trial) + { + // Reset model before using for next trial. + model.Reset(); + MomentumSGD opt(0.06, 50, 2, -50000); + + arma::cube inputTemp, labelsTemp; + for (size_t iteration = 0; iteration < (iterations + offset); iteration++) + { + for (size_t j = 0; j < trainReberGrammarCount; ++j) + { + // Each sequence may be a different length, so we need to extract them + // manually. We will reshape them into a cube with each slice equal to + // a time step. + inputTemp = arma::cube(trainInput.at(0, j).memptr(), inputSize, 1, + trainInput.at(0, j).n_elem / inputSize, false, true); + labelsTemp = arma::cube(trainLabels.at(0, j).memptr(), inputSize, 1, + trainInput.at(0, j).n_elem / inputSize, false, true); + + model.Rho() = inputTemp.n_elem / inputSize; + model.Train(inputTemp, labelsTemp, opt); + opt.ResetPolicy() = false; + } + } + + double error = 0; + + // Ask the network to predict the next Reber grammar in the given sequence. + for (size_t i = 0; i < testReberGrammarCount; ++i) + { + arma::cube prediction; + arma::cube input(testInput.at(0, i).memptr(), inputSize, 1, + testInput.at(0, i).n_elem / inputSize, false, true); + + model.Rho() = input.n_elem / inputSize; + model.Predict(input, prediction); + + const size_t reberGrammerSize = 7; + std::string inputReber = ""; + + size_t reberError = 0; + + for (size_t j = 0; j < (prediction.n_elem / reberGrammerSize); ++j) + { + char predictedSymbol, inputSymbol; + std::string reberChoices; + + arma::umat output = (prediction.slice(j) == (arma::ones( + reberGrammerSize, 1) * + arma::as_scalar(arma::max(prediction.slice(j))))); + + ReberReverseTranslation(output, predictedSymbol); + ReberReverseTranslation(input.slice(j), inputSymbol); + inputReber += inputSymbol; + + if (recursive) + GenerateNextRecursiveReber(transitions, inputReber, reberChoices); + else + GenerateNextReber(transitions, inputReber, reberChoices); + + if (reberChoices.find(predictedSymbol) != std::string::npos) + reberError++; + } + + if (reberError != (prediction.n_elem / reberGrammerSize)) + error += 1; + } + + error /= testReberGrammarCount; + if (error <= 0.3) + { + ++successes; + break; + } + + offset += 3; + } + + REQUIRE(successes >= 1); +} + +/** + * Train the specified networks on an embedded Reber grammar dataset. + */ +TEST_CASE("LSTMReberGrammarTest", "[RecurrentNetworkTest]") +{ + RNN > model(5); + model.Add >(7, 10); + model.Add >(10, 10); + model.Add >(10, 7); + model.Add >(); + ReberGrammarTestNetwork(model, false); +} + +/** + * Train the specified networks on an embedded Reber grammar dataset. + */ +TEST_CASE("FastLSTMReberGrammarTest", "[RecurrentNetworkTest]") +{ + RNN > model(5); + model.Add >(7, 8); + model.Add >(8, 8); + model.Add >(8, 7); + model.Add >(); + ReberGrammarTestNetwork(model, false); +} + +/** + * Train the specified networks on an embedded Reber grammar dataset. + */ +TEST_CASE("GRURecursiveReberGrammarTest", "[RecurrentNetworkTest]") +{ + RNN > model(5); + model.Add >(7, 16); + model.Add >(16, 16); + model.Add >(16, 7); + model.Add >(); + ReberGrammarTestNetwork(model, true, 3, 5, 10, 7); +} + +/** + * Train BLSTM on an embedded Reber grammar dataset. + */ +TEST_CASE("BRNNReberGrammarTest", "[RecurrentNetworkTest]") +{ + BRNN, AddMerge<>, SigmoidLayer<> > model(5); + model.Add >(7, 10); + model.Add >(10, 10); + model.Add >(10, 7); + ReberGrammarTestNetwork(model, false, 3, 5, 1); +} + +/** + * Test RNN with a custom layer. + */ +void ReberGrammarTestCustomNetwork(const size_t hiddenSize = 4, + const bool recursive = false, + const size_t iterations = 10) +{ + const size_t trainReberGrammarCount = 700; + const size_t testReberGrammarCount = 250; + + arma::field trainInput(1, trainReberGrammarCount); + arma::field trainLabels(1, trainReberGrammarCount); + arma::field testInput(1, testReberGrammarCount); + + arma::Mat transitions = + GenerateReberGrammarData(trainInput, + trainLabels, + testInput, + recursive, + trainReberGrammarCount, + testReberGrammarCount); + + /* + * Construct a network with 7 input units, layerSize hidden units and 7 output + * units. The hidden layer is connected to itself. The network structure looks + * like: + * + * Input Hidden Output + * Layer(7) Layer(layerSize) Layer(7) + * +-----+ +-----+ +-----+ + * | | | | | | + * | +------>| +------>| | + * | | ..>| | | | + * +-----+ . +--+--+ +-- ---+ + * . . + * . . + * ....... + */ + // It isn't guaranteed that the recurrent network will converge in the + // specified number of iterations using random weights. If this works 1 of 10 + // times, I'm fine with that. All I want to know is that the network is able + // to escape from local minima and to solve the task. + size_t successes = 0; + size_t offset = 0; + for (size_t trial = 0; trial < 10; ++trial) + { + const size_t outputSize = 7; + const size_t inputSize = 7; + + RNN, RandomInitialization, CustomLayer<> > model(5); + model.Add >(inputSize, hiddenSize); + model.Add >(hiddenSize, hiddenSize); + model.Add >(hiddenSize, outputSize); + model.Add >(); + MomentumSGD opt(0.06, 50, 2, -50000); + + arma::cube inputTemp, labelsTemp; + for (size_t iteration = 0; iteration < (iterations + offset); iteration++) + { + for (size_t j = 0; j < trainReberGrammarCount; ++j) + { + // Each sequence may be a different length, so we need to extract them + // manually. We will reshape them into a cube with each slice equal to + // a time step. + inputTemp = arma::cube(trainInput.at(0, j).memptr(), inputSize, 1, + trainInput.at(0, j).n_elem / inputSize, false, true); + labelsTemp = arma::cube(trainLabels.at(0, j).memptr(), inputSize, 1, + trainInput.at(0, j).n_elem / inputSize, false, true); + + model.Rho() = inputTemp.n_elem / inputSize; + model.Train(inputTemp, labelsTemp, opt); + opt.ResetPolicy() = false; + } + } + + double error = 0; + + // Ask the network to predict the next Reber grammar in the given sequence. + for (size_t i = 0; i < testReberGrammarCount; ++i) + { + arma::cube prediction; + arma::cube input(testInput.at(0, i).memptr(), inputSize, 1, + testInput.at(0, i).n_elem / inputSize, false, true); + + model.Rho() = input.n_elem / inputSize; + model.Predict(input, prediction); + + const size_t reberGrammerSize = 7; + std::string inputReber = ""; + + size_t reberError = 0; + + for (size_t j = 0; j < (prediction.n_elem / reberGrammerSize); ++j) + { + char predictedSymbol, inputSymbol; + std::string reberChoices; + + arma::umat output = (prediction.slice(j) == (arma::ones( + reberGrammerSize, 1) * + arma::as_scalar(arma::max(prediction.slice(j))))); + + ReberReverseTranslation(output, predictedSymbol); + ReberReverseTranslation(input.slice(j), inputSymbol); + inputReber += inputSymbol; + + if (recursive) + GenerateNextRecursiveReber(transitions, inputReber, reberChoices); + else + GenerateNextReber(transitions, inputReber, reberChoices); + + if (reberChoices.find(predictedSymbol) != std::string::npos) + reberError++; + } + + if (reberError != (prediction.n_elem / reberGrammerSize)) + error += 1; + } + + error /= testReberGrammarCount; + if (error <= 0.35) + { + ++successes; + break; + } + + offset += 3; + } + + REQUIRE(successes >= 1); +} + +/** + * Train the specified networks on an embedded Reber grammar dataset. + */ +TEST_CASE("CustomRecursiveReberGrammarTest", "[RecurrentNetworkTest]") +{ + ReberGrammarTestCustomNetwork(16, true); +}