From 17f4a934d0fe227e0bfa074add030fca6561fbdd Mon Sep 17 00:00:00 2001 From: Omar Shrit Date: Fri, 2 Oct 2020 22:56:30 +0200 Subject: [PATCH] Split up recurrent_network_test Signed-off-by: Omar Shrit --- src/mlpack/tests/CMakeLists.txt | 1 + src/mlpack/tests/recurrent_network_2_test.cpp | 806 ++++++++++++++++++ src/mlpack/tests/recurrent_network_test.cpp | 776 ----------------- 3 files changed, 807 insertions(+), 776 deletions(-) create mode 100644 src/mlpack/tests/recurrent_network_2_test.cpp diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index aa6b4b7e64..c9aa63ca03 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -150,6 +150,7 @@ add_executable(mlpack_catch_test randomized_svd_test.cpp rbm_network_test.cpp recurrent_network_test.cpp + recurrent_network_2_test.cpp regularized_svd_test.cpp scaling_test.cpp serialization_catch.cpp diff --git a/src/mlpack/tests/recurrent_network_2_test.cpp b/src/mlpack/tests/recurrent_network_2_test.cpp new file mode 100644 index 0000000000..8187ed4c4b --- /dev/null +++ b/src/mlpack/tests/recurrent_network_2_test.cpp @@ -0,0 +1,806 @@ +/** + * @file tests/recurrent_network_test.cpp + * @author Marcus Edel + * + * Tests the recurrent network. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "catch.hpp" +#include "serialization_catch.hpp" +#include "custom_layer.hpp" + +using namespace mlpack; +using namespace mlpack::ann; +using namespace ens; +using namespace mlpack::math; + +/* + * This sample is a simplified version of Derek D. Monner's Distracted Sequence + * Recall task, which involves 10 symbols: + * + * Targets: must be recognized and remembered by the network. + * Distractors: never need to be remembered. + * Prompts: direct the network to give an answer. + * + * A single trial consists of a temporal sequence of 10 input symbols. The first + * 8 consist of 2 randomly chosen target symbols and 6 randomly chosen + * distractor symbols in an random order. The remaining two symbols are two + * prompts, which direct the network to produce the first and second target in + * the sequence, in order. + * + * For more information, see the following paper. + * + * @code + * @misc{Monner2012, + * author = {Monner, Derek and Reggia, James A}, + * title = {A generalized LSTM-like training algorithm for second-order + * recurrent neural networks}, + * year = {2012} + * } + * @endcode + * + * @param input The generated input sequence. + * @param input The generated output sequence. + */ +void GenerateDistractedSequence(arma::mat& input, arma::mat& output) +{ + input = arma::zeros(10, 10); + output = arma::zeros(3, 10); + + arma::uvec index = arma::shuffle(arma::linspace(0, 7, 8)); + + // Set the target in the input sequence and the corresponding targets in the + // output sequence by following the correct order. + for (size_t i = 0; i < 2; ++i) + { + size_t idx = rand() % 2; + input(idx, index(i)) = 1; + output(idx, index(i) > index(i == 0) ? 9 : 8) = 1; + } + + for (size_t i = 2; i < 8; ++i) + input(2 + rand() % 6, index(i)) = 1; + + // Set the prompts which direct the network to give an answer. + input(8, 8) = 1; + input(9, 9) = 1; + + input.reshape(input.n_elem, 1); + output.reshape(output.n_elem, 1); +} + +/** + * Train the specified network and the construct distracted sequence recall + * dataset. + */ +template +void DistractedSequenceRecallTestNetwork( + const size_t cellSize, const size_t hiddenSize) +{ + const size_t trainDistractedSequenceCount = 600; + const size_t testDistractedSequenceCount = 300; + + arma::field trainInput(1, trainDistractedSequenceCount); + arma::field trainLabels(1, trainDistractedSequenceCount); + arma::field testInput(1, testDistractedSequenceCount); + arma::field testLabels(1, testDistractedSequenceCount); + + // Generate the training data. + for (size_t i = 0; i < trainDistractedSequenceCount; ++i) + GenerateDistractedSequence(trainInput(0, i), trainLabels(0, i)); + + // Generate the test data. + for (size_t i = 0; i < testDistractedSequenceCount; ++i) + GenerateDistractedSequence(testInput(0, i), testLabels(0, i)); + + /* + * Construct a network with 10 input units, layerSize hidden units and 3 + * output units. The hidden layer is connected to itself. The network + * structure looks like: + * + * Input Recurrent Hidden Output + * Layer(10) Layer(cellSize) Layer(3) Layer(3) + * +-----+ +-----+ +-----+ +-----+ + * | | | | | | | | + * | +------>| +------>| |------>| | + * | | ..>| | | | | | + * +-----+ . +--+--+ +-----+ +-----+ + * . . + * . . + * ....... + */ + const size_t outputSize = 3; + const size_t inputSize = 10; + const size_t rho = trainInput.at(0, 0).n_elem / inputSize; + + // It isn't guaranteed that the recurrent network will converge in the + // specified number of iterations using random weights. If this works 1 of 5 + // times, I'm fine with that. All I want to know is that the network is able + // to escape from local minima and to solve the task. + size_t successes = 0; + size_t offset = 0; + for (size_t trial = 0; trial < 5; ++trial) + { + RNN > model(rho); + model.Add >(); + model.Add >(inputSize, cellSize); + model.Add(cellSize, hiddenSize); + model.Add >(hiddenSize, outputSize); + model.Add >(); + + StandardSGD opt(0.1, 50, 2, -50000); + + // We increase the number of iterations (training) if the first run didn't + // pass. + arma::cube inputTemp, labelsTemp; + for (size_t iteration = 0; iteration < (9 + offset); iteration++) + { + for (size_t j = 0; j < trainDistractedSequenceCount; ++j) + { + inputTemp = arma::cube(trainInput.at(0, j).memptr(), inputSize, 1, + trainInput.at(0, j).n_elem / inputSize, false, true); + labelsTemp = arma::cube(trainLabels.at(0, j).memptr(), outputSize, 1, + trainLabels.at(0, j).n_elem / outputSize, false, true); + + model.Train(inputTemp, labelsTemp, opt); + } + } + + double error = 0; + + // Ask the network to predict the targets in the given sequence at the + // prompts. + for (size_t i = 0; i < testDistractedSequenceCount; ++i) + { + arma::cube output; + arma::cube input(testInput.at(0, i).memptr(), inputSize, 1, + testInput.at(0, i).n_elem / inputSize, false, true); + + model.Predict(input, output); + for (size_t j = 0; j < output.n_slices; ++j) + { + arma::mat outputSlice = output.slice(j); + data::Binarize(outputSlice, outputSlice, 0.5); + output.slice(j) = outputSlice; + } + + arma::cube label(testLabels.at(0, i).memptr(), outputSize, 1, + testLabels.at(0, i).n_elem / outputSize, false, true); + if (arma::accu(arma::abs(label - output)) != 0) + error += 1; + } + + error /= testDistractedSequenceCount; + // Can we reproduce the results from the paper. They provide an 95% accuracy + // on a test set of 1000 randomly selected sequences. + // Ensure that this is within tolerance, which is at least as good as the + // paper's results (plus a little bit for noise). + if (error <= 0.3) + { + ++successes; + break; + } + + offset += 2; + } + + REQUIRE(successes >= 1); +} + +/** + * Train the specified networks on the Derek D. Monner's distracted sequence + * recall task. + */ +TEST_CASE("LSTMDistractedSequenceRecallTest", "[RecurrentNetworkTest]") +{ + DistractedSequenceRecallTestNetwork >(4, 8); +} + +/** + * Train the specified networks on the Derek D. Monner's distracted sequence + * recall task. + */ +TEST_CASE("FastLSTMDistractedSequenceRecallTest", "[RecurrentNetworkTest]") +{ + DistractedSequenceRecallTestNetwork >(4, 8); +} + +/** + * Train the specified networks on the Derek D. Monner's distracted sequence + * recall task. + */ +TEST_CASE("GRUDistractedSequenceRecallTest", "[RecurrentNetworkTest]") +{ + DistractedSequenceRecallTestNetwork >(4, 8); +} + +/** + * Create a simple recurrent neural network for the noisy sines task, and + * require that it produces the exact same network for a few batch sizes. + */ +template +void BatchSizeTest() +{ + const size_t rho = 10; + + // Generate 12 (2 * 6) noisy sines. A single sine contains rho + // points/features. + arma::cube input; + arma::mat labelsTemp; + GenerateNoisySines(input, labelsTemp, rho, 6); + + arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); + for (size_t i = 0; i < labelsTemp.n_cols; ++i) + { + const int value = arma::as_scalar(arma::find( + arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; + labels.tube(0, i).fill(value); + } + + RNN<> model(rho); + model.Add>(1, 10); + model.Add>(); + model.Add(10, 10); + model.Add>(); + model.Add>(10, 10); + model.Add>(); + + model.Reset(); + arma::mat initParams = model.Parameters(); + + StandardSGD opt(1e-5, 1, 5, -100, false); + model.Train(input, labels, opt); + + // This is trained with one point. + arma::mat outputParams = model.Parameters(); + + model.Reset(); + model.Parameters() = initParams; + opt.BatchSize() = 2; + model.Train(input, labels, opt); + + CheckMatrices(outputParams, model.Parameters(), 1); + + model.Parameters() = initParams; + opt.BatchSize() = 5; + model.Train(input, labels, opt); + + CheckMatrices(outputParams, model.Parameters(), 1); +} + +/** + * Ensure LSTMs work with larger batch sizes. + */ +TEST_CASE("LSTMBatchSizeTest", "[RecurrentNetworkTest]") +{ + BatchSizeTest>(); +} + +/** + * Ensure fast LSTMs work with larger batch sizes. + */ +TEST_CASE("FastLSTMBatchSizeTest", "[RecurrentNetworkTest]") +{ + BatchSizeTest>(); +} + +/** + * Ensure GRUs work with larger batch sizes. + */ +TEST_CASE("GRUBatchSizeTest", "[RecurrentNetworkTest]") +{ + BatchSizeTest>(); +} + +/** + * Make sure the RNN can be properly serialized. + */ +TEST_CASE("RNNSerializationTest", "[RecurrentNetworkTest]") +{ + const size_t rho = 10; + + // Generate 12 (2 * 6) noisy sines. A single sine contains rho + // points/features. + arma::cube input; + arma::mat labelsTemp; + GenerateNoisySines(input, labelsTemp, rho, 6); + + arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); + for (size_t i = 0; i < labelsTemp.n_cols; ++i) + { + const int value = arma::as_scalar(arma::find( + arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; + labels.tube(0, i).fill(value); + } + + /** + * Construct a network with 1 input unit, 4 hidden units and 10 output + * units. The hidden layer is connected to itself. The network structure + * looks like: + * + * Input Hidden Output + * Layer(1) Layer(4) Layer(10) + * +-----+ +-----+ +-----+ + * | | | | | | + * | +------>| +------>| | + * | | ..>| | | | + * +-----+ . +--+--+ +-----+ + * . . + * . . + * ....... + */ + Add<> add(4); + Linear<> lookup(1, 4); + SigmoidLayer<> sigmoidLayer; + Linear<> linear(4, 4); + Recurrent<>* recurrent = new Recurrent<>(add, lookup, linear, + sigmoidLayer, rho); + + RNN<> model(rho); + model.Add >(); + model.Add(recurrent); + model.Add >(4, 10); + model.Add >(); + + StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch */, -100); + model.Train(input, labels, opt); + + // Serialize the network. + RNN<> xmlModel(1), jsonModel(3), binaryModel(5); + SerializeObjectAll(model, xmlModel, jsonModel, binaryModel); + + // Take predictions, check the output. + arma::cube prediction, xmlPrediction, jsonPrediction, binaryPrediction; + model.Predict(input, prediction); + xmlModel.Predict(input, xmlPrediction); + jsonModel.Predict(input, jsonPrediction); + binaryModel.Predict(input, binaryPrediction); + + CheckMatrices(prediction, xmlPrediction, jsonPrediction, binaryPrediction); +} + +/** + * Test RNN with a custom layer. + */ +void ReberGrammarTestCustomNetwork(const size_t hiddenSize = 4, + const bool recursive = false, + const size_t iterations = 10) +{ + const size_t trainReberGrammarCount = 700; + const size_t testReberGrammarCount = 250; + + arma::field trainInput(1, trainReberGrammarCount); + arma::field trainLabels(1, trainReberGrammarCount); + arma::field testInput(1, testReberGrammarCount); + + arma::Mat transitions = + GenerateReberGrammarData(trainInput, + trainLabels, + testInput, + recursive, + trainReberGrammarCount, + testReberGrammarCount); + + /* + * Construct a network with 7 input units, layerSize hidden units and 7 output + * units. The hidden layer is connected to itself. The network structure looks + * like: + * + * Input Hidden Output + * Layer(7) Layer(layerSize) Layer(7) + * +-----+ +-----+ +-----+ + * | | | | | | + * | +------>| +------>| | + * | | ..>| | | | + * +-----+ . +--+--+ +-- ---+ + * . . + * . . + * ....... + */ + // It isn't guaranteed that the recurrent network will converge in the + // specified number of iterations using random weights. If this works 1 of 10 + // times, I'm fine with that. All I want to know is that the network is able + // to escape from local minima and to solve the task. + size_t successes = 0; + size_t offset = 0; + for (size_t trial = 0; trial < 10; ++trial) + { + const size_t outputSize = 7; + const size_t inputSize = 7; + + RNN, RandomInitialization, CustomLayer<> > model(5); + model.Add >(inputSize, hiddenSize); + model.Add >(hiddenSize, hiddenSize); + model.Add >(hiddenSize, outputSize); + model.Add >(); + MomentumSGD opt(0.06, 50, 2, -50000); + + arma::cube inputTemp, labelsTemp; + for (size_t iteration = 0; iteration < (iterations + offset); iteration++) + { + for (size_t j = 0; j < trainReberGrammarCount; ++j) + { + // Each sequence may be a different length, so we need to extract them + // manually. We will reshape them into a cube with each slice equal to + // a time step. + inputTemp = arma::cube(trainInput.at(0, j).memptr(), inputSize, 1, + trainInput.at(0, j).n_elem / inputSize, false, true); + labelsTemp = arma::cube(trainLabels.at(0, j).memptr(), inputSize, 1, + trainInput.at(0, j).n_elem / inputSize, false, true); + + model.Rho() = inputTemp.n_elem / inputSize; + model.Train(inputTemp, labelsTemp, opt); + opt.ResetPolicy() = false; + } + } + + double error = 0; + + // Ask the network to predict the next Reber grammar in the given sequence. + for (size_t i = 0; i < testReberGrammarCount; ++i) + { + arma::cube prediction; + arma::cube input(testInput.at(0, i).memptr(), inputSize, 1, + testInput.at(0, i).n_elem / inputSize, false, true); + + model.Rho() = input.n_elem / inputSize; + model.Predict(input, prediction); + + const size_t reberGrammerSize = 7; + std::string inputReber = ""; + + size_t reberError = 0; + + for (size_t j = 0; j < (prediction.n_elem / reberGrammerSize); ++j) + { + char predictedSymbol, inputSymbol; + std::string reberChoices; + + arma::umat output = (prediction.slice(j) == (arma::ones( + reberGrammerSize, 1) * + arma::as_scalar(arma::max(prediction.slice(j))))); + + ReberReverseTranslation(output, predictedSymbol); + ReberReverseTranslation(input.slice(j), inputSymbol); + inputReber += inputSymbol; + + if (recursive) + GenerateNextRecursiveReber(transitions, inputReber, reberChoices); + else + GenerateNextReber(transitions, inputReber, reberChoices); + + if (reberChoices.find(predictedSymbol) != std::string::npos) + reberError++; + } + + if (reberError != (prediction.n_elem / reberGrammerSize)) + error += 1; + } + + error /= testReberGrammarCount; + if (error <= 0.35) + { + ++successes; + break; + } + + offset += 3; + } + + REQUIRE(successes >= 1); +} + +/** + * Train the specified networks on an embedded Reber grammar dataset. + */ +TEST_CASE("CustomRecursiveReberGrammarTest", "[RecurrentNetworkTest]") +{ + ReberGrammarTestCustomNetwork(16, true); +} + +/** + * @brief Generates noisy sine wave and outputs the data and the labels that + * can be used directly for training and testing with RNN. + * + * @param data The data points as output + * @param labels The expected values as output + * @param rho The size of the sequence of each data point + * @param outputSteps How many output steps to consider for every rho inputs + * @param dataPoints The number of generated data points. The actual generated + * data points may be more than this to adjust to the outputSteps. But at + * the minimum these many data points will be generated. + * @param gain The gain on the amplitude + * @param freq The frquency of the sine wave + * @param phase The phase shift if any + * @param noisePercent The percent noise to induce + * @param numCycles How many full size wave cycles required. All the data + * points will be fit into these cycles. + * @param normalize Whether to normalise the data. This may be required for some + * layers like LSTM. Default is true. + */ +void GenerateNoisySinRNN(arma::cube& data, + arma::cube& labels, + size_t rho, + size_t outputSteps = 1, + const int dataPoints = 100, + const double gain = 1.0, + const int freq = 10, + const double phase = 0, + const int noisePercent = 20, + const double numCycles = 6.0, + const bool normalize = true) +{ + int points = dataPoints; + int r = dataPoints % rho; + + if (r == 0) + { + points += outputSteps; + } + else + { + points += rho - r + outputSteps; + } + + arma::colvec x(points); + int i = 0; + double interval = numCycles / freq / points; + + x.for_each([&i, gain, freq, phase, noisePercent, interval] + (arma::colvec::elem_type& val) { + double t = interval * (++i); + val = gain * ::sin(2 * M_PI * freq * t + phase) + + (noisePercent * gain / 100 * Random(0.0, 0.1)); + }); + + arma::colvec y = x; + if (normalize) + y = arma::normalise(x); + + // Now break this into columns of rho size slices. + size_t numColumns = y.n_elem / rho; + data = arma::cube(1, numColumns, rho); + labels = arma::cube(outputSteps, numColumns, 1); + + for (size_t i = 0; i < numColumns; ++i) + { + data.tube(0, i) = y.rows(i * rho, i * rho + rho - 1); + labels.subcube(0, i, 0, outputSteps - 1, i, 0) = + y.rows(i * rho + rho, i * rho + rho + outputSteps - 1); + } +} + +/** + * @brief RNNSineTest Test a simple RNN using noisy sine. Use single output + * for multiple inputs. + * @param hiddenUnits No of units in the hiddenlayer. + * @param rho The input sequence length. + * @param numEpochs The number of epochs to run. + * @return The mean squared error of the prediction. + */ +double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100) +{ + RNN > net(rho, true); + net.Add >(1, hiddenUnits); + net.Add >(hiddenUnits, hiddenUnits); + net.Add >(hiddenUnits, 1); + + RMSProp opt(0.005, 100, 0.9, 1e-08, 50000, 1e-5); + + // Generate data + arma::cube data; + arma::cube labels; + GenerateNoisySinRNN(data, labels, rho, 1, 2000, 20.0, 200, 0.0, 45, 20); + + // Break into training and test sets. Simply split along columns. + size_t trainCols = data.n_cols * 0.8; // Take 20% out for testing. + size_t testCols = data.n_cols - trainCols; + arma::cube testData = data.subcube(0, data.n_cols - testCols, 0, + data.n_rows - 1, data.n_cols - 1, data.n_slices - 1); + arma::cube testLabels = labels.subcube(0, labels.n_cols - testCols, 0, + labels.n_rows - 1, labels.n_cols - 1, labels.n_slices - 1); + + for (size_t i = 0; i < numEpochs; ++i) + { + net.Train(data.subcube(0, 0, 0, data.n_rows - 1, trainCols - 1, + data.n_slices - 1), labels.subcube(0, 0, 0, labels.n_rows - 1, + trainCols - 1, labels.n_slices - 1), opt); + } + // Well now it should be trained. Do the test here. + arma::cube prediction; + net.Predict(testData, prediction); + + // The prediction must really follow the test data. So convert both the test + // data and the pediction to vectors and compare the two. + arma::colvec testVector = arma::vectorise(testData); + arma::colvec predVector = arma::vectorise(prediction); + + // Adjust the vectors for comparison, as the prediction is one step ahead. + testVector = testVector.rows(1, testVector.n_rows - 1); + predVector = predVector.rows(0, predVector.n_rows - 2); + double error = std::sqrt(arma::sum(arma::square(testVector - predVector))) / + testVector.n_rows; + + return error; +} + +/** + * Test RNN using multiple timestep input and single output. + */ +TEST_CASE("MultiTimestepTest", "[RecurrentNetworkTest]") +{ + double err = RNNSineTest(4, 10, 20); + REQUIRE(err <= 0.025); +} + +/** + * Test that RNN::Train() returns finite objective value. + */ +TEST_CASE("RNNTrainReturnObjective", "[RecurrentNetworkTest]") +{ + const size_t rho = 10; + + // Generate 12 (2 * 6) noisy sines. A single sine contains rho + // points/features. + arma::cube input; + arma::mat labelsTemp; + GenerateNoisySines(input, labelsTemp, rho, 6); + + arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); + for (size_t i = 0; i < labelsTemp.n_cols; ++i) + { + const int value = arma::as_scalar(arma::find( + arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; + labels.tube(0, i).fill(value); + } + + /** + * Construct a network with 1 input unit, 4 hidden units and 10 output + * units. The hidden layer is connected to itself. The network structure + * looks like: + * + * Input Hidden Output + * Layer(1) Layer(4) Layer(10) + * +-----+ +-----+ +-----+ + * | | | | | | + * | +------>| +------>| | + * | | ..>| | | | + * +-----+ . +--+--+ +-----+ + * . . + * . . + * ....... + */ + Add<> add(4); + Linear<> lookup(1, 4); + SigmoidLayer<> sigmoidLayer; + Linear<> linear(4, 4); + Recurrent<>* recurrent = new Recurrent<>(add, lookup, linear, + sigmoidLayer, rho); + + RNN<> model(rho); + model.Add >(); + model.Add(recurrent); + model.Add >(4, 10); + model.Add >(); + + StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch */, -100); + double objVal = model.Train(input, labels, opt); + + REQUIRE(std::isfinite(objVal) == true); +} + +/** + * Test that BRNN::Train() returns finite objective value. + */ +TEST_CASE("BRNNTrainReturnObjective", "[RecurrentNetworkTest]") +{ + const size_t rho = 10; + + arma::cube input; + arma::mat labelsTemp; + GenerateNoisySines(input, labelsTemp, rho, 6); + + arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); + for (size_t i = 0; i < labelsTemp.n_cols; ++i) + { + const int value = arma::as_scalar(arma::find( + arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; + labels.tube(0, i).fill(value); + } + + Add<> add(4); + Linear<> lookup(1, 4); + SigmoidLayer<> sigmoidLayer; + Linear<> linear(4, 4); + Recurrent<>* recurrent = new Recurrent<>( + add, lookup, linear, sigmoidLayer, rho); + + BRNN<> model(rho); + model.Add >(); + model.Add(recurrent); + model.Add >(4, 5); + + StandardSGD opt(0.1, 1, 500 * input.n_cols, -100); + double objVal = model.Train(input, labels, opt); + INFO("Training over"); + + // Test that BRNN::Train() returns finite objective value. + REQUIRE(std::isfinite(objVal) == true); +} + +/** + * Test that RNN::Train() does not give an error for large rho. + */ +TEST_CASE("LargeRhoValueRnnTest", "[RecurrentNetworkTest]") +{ + // Setting rho value greater than sequence length which is 17. + const size_t rho = 100; + const size_t hiddenSize = 128; + const size_t numLetters = 256; + using MatType = arma::cube; + std::vectortrainingData = { "THIS IS THE INPUT 0" , + "THIS IS THE INPUT 1" , + "THIS IS THE INPUT 3"}; + + + RNN<> model(rho); + model.Add>(); + model.Add>(numLetters, hiddenSize, rho); + model.Add>(0.1); + model.Add>(hiddenSize, numLetters); + + const auto makeInput = [numLetters](const char *line) -> MatType + { + const auto strLen = strlen(line); + // Rows: number of dimensions. + // Cols: number of sequences/points. + // Slices: number of steps in sequences. + MatType result(numLetters, 1, strLen, arma::fill::zeros); + for (size_t i = 0; i < strLen; ++i) + { + result.at(static_cast(line[i]), 0, i) = 1.0; + } + return result; + }; + + const auto makeTarget = [] (const char *line) -> MatType + { + const auto strLen = strlen(line); + // Responses for NegativeLogLikelihood should be + // non-one-hot-encoded class IDs (from 1 to num_classes). + MatType result(1, 1, strLen, arma::fill::zeros); + // The response is the *next* letter in the sequence. + for (size_t i = 0; i < strLen - 1; ++i) + { + result.at(0, 0, i) = static_cast(line[i + 1]) + 1.0; + } + // The final response is empty, so we set it to class 0. + result.at(0, 0, strLen - 1) = 1.0; + return result; + }; + + std::vector inputs(trainingData.size()); + std::vector targets(trainingData.size()); + for (size_t i = 0; i < trainingData.size(); ++i) + { + inputs[i] = makeInput(trainingData[i].c_str()); + targets[i] = makeTarget(trainingData[i].c_str()); + } + ens::SGD<> opt(0.01, 1, 100); + model.Train(inputs[0], targets[0], opt); + INFO("Training over"); +} diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp index eb687190ae..7a41b49df7 100644 --- a/src/mlpack/tests/recurrent_network_test.cpp +++ b/src/mlpack/tests/recurrent_network_test.cpp @@ -696,779 +696,3 @@ TEST_CASE("BRNNReberGrammarTest", "[RecurrentNetworkTest]") ReberGrammarTestNetwork(model, false, 3, 5, 1); } -/* - * This sample is a simplified version of Derek D. Monner's Distracted Sequence - * Recall task, which involves 10 symbols: - * - * Targets: must be recognized and remembered by the network. - * Distractors: never need to be remembered. - * Prompts: direct the network to give an answer. - * - * A single trial consists of a temporal sequence of 10 input symbols. The first - * 8 consist of 2 randomly chosen target symbols and 6 randomly chosen - * distractor symbols in an random order. The remaining two symbols are two - * prompts, which direct the network to produce the first and second target in - * the sequence, in order. - * - * For more information, see the following paper. - * - * @code - * @misc{Monner2012, - * author = {Monner, Derek and Reggia, James A}, - * title = {A generalized LSTM-like training algorithm for second-order - * recurrent neural networks}, - * year = {2012} - * } - * @endcode - * - * @param input The generated input sequence. - * @param input The generated output sequence. - */ -void GenerateDistractedSequence(arma::mat& input, arma::mat& output) -{ - input = arma::zeros(10, 10); - output = arma::zeros(3, 10); - - arma::uvec index = arma::shuffle(arma::linspace(0, 7, 8)); - - // Set the target in the input sequence and the corresponding targets in the - // output sequence by following the correct order. - for (size_t i = 0; i < 2; ++i) - { - size_t idx = rand() % 2; - input(idx, index(i)) = 1; - output(idx, index(i) > index(i == 0) ? 9 : 8) = 1; - } - - for (size_t i = 2; i < 8; ++i) - input(2 + rand() % 6, index(i)) = 1; - - // Set the prompts which direct the network to give an answer. - input(8, 8) = 1; - input(9, 9) = 1; - - input.reshape(input.n_elem, 1); - output.reshape(output.n_elem, 1); -} - -/** - * Train the specified network and the construct distracted sequence recall - * dataset. - */ -template -void DistractedSequenceRecallTestNetwork( - const size_t cellSize, const size_t hiddenSize) -{ - const size_t trainDistractedSequenceCount = 600; - const size_t testDistractedSequenceCount = 300; - - arma::field trainInput(1, trainDistractedSequenceCount); - arma::field trainLabels(1, trainDistractedSequenceCount); - arma::field testInput(1, testDistractedSequenceCount); - arma::field testLabels(1, testDistractedSequenceCount); - - // Generate the training data. - for (size_t i = 0; i < trainDistractedSequenceCount; ++i) - GenerateDistractedSequence(trainInput(0, i), trainLabels(0, i)); - - // Generate the test data. - for (size_t i = 0; i < testDistractedSequenceCount; ++i) - GenerateDistractedSequence(testInput(0, i), testLabels(0, i)); - - /* - * Construct a network with 10 input units, layerSize hidden units and 3 - * output units. The hidden layer is connected to itself. The network - * structure looks like: - * - * Input Recurrent Hidden Output - * Layer(10) Layer(cellSize) Layer(3) Layer(3) - * +-----+ +-----+ +-----+ +-----+ - * | | | | | | | | - * | +------>| +------>| |------>| | - * | | ..>| | | | | | - * +-----+ . +--+--+ +-----+ +-----+ - * . . - * . . - * ....... - */ - const size_t outputSize = 3; - const size_t inputSize = 10; - const size_t rho = trainInput.at(0, 0).n_elem / inputSize; - - // It isn't guaranteed that the recurrent network will converge in the - // specified number of iterations using random weights. If this works 1 of 5 - // times, I'm fine with that. All I want to know is that the network is able - // to escape from local minima and to solve the task. - size_t successes = 0; - size_t offset = 0; - for (size_t trial = 0; trial < 5; ++trial) - { - RNN > model(rho); - model.Add >(); - model.Add >(inputSize, cellSize); - model.Add(cellSize, hiddenSize); - model.Add >(hiddenSize, outputSize); - model.Add >(); - - StandardSGD opt(0.1, 50, 2, -50000); - - // We increase the number of iterations (training) if the first run didn't - // pass. - arma::cube inputTemp, labelsTemp; - for (size_t iteration = 0; iteration < (9 + offset); iteration++) - { - for (size_t j = 0; j < trainDistractedSequenceCount; ++j) - { - inputTemp = arma::cube(trainInput.at(0, j).memptr(), inputSize, 1, - trainInput.at(0, j).n_elem / inputSize, false, true); - labelsTemp = arma::cube(trainLabels.at(0, j).memptr(), outputSize, 1, - trainLabels.at(0, j).n_elem / outputSize, false, true); - - model.Train(inputTemp, labelsTemp, opt); - } - } - - double error = 0; - - // Ask the network to predict the targets in the given sequence at the - // prompts. - for (size_t i = 0; i < testDistractedSequenceCount; ++i) - { - arma::cube output; - arma::cube input(testInput.at(0, i).memptr(), inputSize, 1, - testInput.at(0, i).n_elem / inputSize, false, true); - - model.Predict(input, output); - for (size_t j = 0; j < output.n_slices; ++j) - { - arma::mat outputSlice = output.slice(j); - data::Binarize(outputSlice, outputSlice, 0.5); - output.slice(j) = outputSlice; - } - - arma::cube label(testLabels.at(0, i).memptr(), outputSize, 1, - testLabels.at(0, i).n_elem / outputSize, false, true); - if (arma::accu(arma::abs(label - output)) != 0) - error += 1; - } - - error /= testDistractedSequenceCount; - // Can we reproduce the results from the paper. They provide an 95% accuracy - // on a test set of 1000 randomly selected sequences. - // Ensure that this is within tolerance, which is at least as good as the - // paper's results (plus a little bit for noise). - if (error <= 0.3) - { - ++successes; - break; - } - - offset += 2; - } - - REQUIRE(successes >= 1); -} - -/** - * Train the specified networks on the Derek D. Monner's distracted sequence - * recall task. - */ -TEST_CASE("LSTMDistractedSequenceRecallTest", "[RecurrentNetworkTest]") -{ - DistractedSequenceRecallTestNetwork >(4, 8); -} - -/** - * Train the specified networks on the Derek D. Monner's distracted sequence - * recall task. - */ -TEST_CASE("FastLSTMDistractedSequenceRecallTest", "[RecurrentNetworkTest]") -{ - DistractedSequenceRecallTestNetwork >(4, 8); -} - -/** - * Train the specified networks on the Derek D. Monner's distracted sequence - * recall task. - */ -TEST_CASE("GRUDistractedSequenceRecallTest", "[RecurrentNetworkTest]") -{ - DistractedSequenceRecallTestNetwork >(4, 8); -} - -/** - * Create a simple recurrent neural network for the noisy sines task, and - * require that it produces the exact same network for a few batch sizes. - */ -template -void BatchSizeTest() -{ - const size_t rho = 10; - - // Generate 12 (2 * 6) noisy sines. A single sine contains rho - // points/features. - arma::cube input; - arma::mat labelsTemp; - GenerateNoisySines(input, labelsTemp, rho, 6); - - arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); - for (size_t i = 0; i < labelsTemp.n_cols; ++i) - { - const int value = arma::as_scalar(arma::find( - arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; - labels.tube(0, i).fill(value); - } - - RNN<> model(rho); - model.Add>(1, 10); - model.Add>(); - model.Add(10, 10); - model.Add>(); - model.Add>(10, 10); - model.Add>(); - - model.Reset(); - arma::mat initParams = model.Parameters(); - - StandardSGD opt(1e-5, 1, 5, -100, false); - model.Train(input, labels, opt); - - // This is trained with one point. - arma::mat outputParams = model.Parameters(); - - model.Reset(); - model.Parameters() = initParams; - opt.BatchSize() = 2; - model.Train(input, labels, opt); - - CheckMatrices(outputParams, model.Parameters(), 1); - - model.Parameters() = initParams; - opt.BatchSize() = 5; - model.Train(input, labels, opt); - - CheckMatrices(outputParams, model.Parameters(), 1); -} - -/** - * Ensure LSTMs work with larger batch sizes. - */ -TEST_CASE("LSTMBatchSizeTest", "[RecurrentNetworkTest]") -{ - BatchSizeTest>(); -} - -/** - * Ensure fast LSTMs work with larger batch sizes. - */ -TEST_CASE("FastLSTMBatchSizeTest", "[RecurrentNetworkTest]") -{ - BatchSizeTest>(); -} - -/** - * Ensure GRUs work with larger batch sizes. - */ -TEST_CASE("GRUBatchSizeTest", "[RecurrentNetworkTest]") -{ - BatchSizeTest>(); -} - -/** - * Make sure the RNN can be properly serialized. - */ -TEST_CASE("RNNSerializationTest", "[RecurrentNetworkTest]") -{ - const size_t rho = 10; - - // Generate 12 (2 * 6) noisy sines. A single sine contains rho - // points/features. - arma::cube input; - arma::mat labelsTemp; - GenerateNoisySines(input, labelsTemp, rho, 6); - - arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); - for (size_t i = 0; i < labelsTemp.n_cols; ++i) - { - const int value = arma::as_scalar(arma::find( - arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; - labels.tube(0, i).fill(value); - } - - /** - * Construct a network with 1 input unit, 4 hidden units and 10 output - * units. The hidden layer is connected to itself. The network structure - * looks like: - * - * Input Hidden Output - * Layer(1) Layer(4) Layer(10) - * +-----+ +-----+ +-----+ - * | | | | | | - * | +------>| +------>| | - * | | ..>| | | | - * +-----+ . +--+--+ +-----+ - * . . - * . . - * ....... - */ - Add<> add(4); - Linear<> lookup(1, 4); - SigmoidLayer<> sigmoidLayer; - Linear<> linear(4, 4); - Recurrent<>* recurrent = new Recurrent<>(add, lookup, linear, - sigmoidLayer, rho); - - RNN<> model(rho); - model.Add >(); - model.Add(recurrent); - model.Add >(4, 10); - model.Add >(); - - StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch */, -100); - model.Train(input, labels, opt); - - // Serialize the network. - RNN<> xmlModel(1), jsonModel(3), binaryModel(5); - SerializeObjectAll(model, xmlModel, jsonModel, binaryModel); - - // Take predictions, check the output. - arma::cube prediction, xmlPrediction, jsonPrediction, binaryPrediction; - model.Predict(input, prediction); - xmlModel.Predict(input, xmlPrediction); - jsonModel.Predict(input, jsonPrediction); - binaryModel.Predict(input, binaryPrediction); - - CheckMatrices(prediction, xmlPrediction, jsonPrediction, binaryPrediction); -} - -/** - * Test RNN with a custom layer. - */ -void ReberGrammarTestCustomNetwork(const size_t hiddenSize = 4, - const bool recursive = false, - const size_t iterations = 10) -{ - const size_t trainReberGrammarCount = 700; - const size_t testReberGrammarCount = 250; - - arma::field trainInput(1, trainReberGrammarCount); - arma::field trainLabels(1, trainReberGrammarCount); - arma::field testInput(1, testReberGrammarCount); - - arma::Mat transitions = - GenerateReberGrammarData(trainInput, - trainLabels, - testInput, - recursive, - trainReberGrammarCount, - testReberGrammarCount); - - /* - * Construct a network with 7 input units, layerSize hidden units and 7 output - * units. The hidden layer is connected to itself. The network structure looks - * like: - * - * Input Hidden Output - * Layer(7) Layer(layerSize) Layer(7) - * +-----+ +-----+ +-----+ - * | | | | | | - * | +------>| +------>| | - * | | ..>| | | | - * +-----+ . +--+--+ +-- ---+ - * . . - * . . - * ....... - */ - // It isn't guaranteed that the recurrent network will converge in the - // specified number of iterations using random weights. If this works 1 of 10 - // times, I'm fine with that. All I want to know is that the network is able - // to escape from local minima and to solve the task. - size_t successes = 0; - size_t offset = 0; - for (size_t trial = 0; trial < 10; ++trial) - { - const size_t outputSize = 7; - const size_t inputSize = 7; - - RNN, RandomInitialization, CustomLayer<> > model(5); - model.Add >(inputSize, hiddenSize); - model.Add >(hiddenSize, hiddenSize); - model.Add >(hiddenSize, outputSize); - model.Add >(); - MomentumSGD opt(0.06, 50, 2, -50000); - - arma::cube inputTemp, labelsTemp; - for (size_t iteration = 0; iteration < (iterations + offset); iteration++) - { - for (size_t j = 0; j < trainReberGrammarCount; ++j) - { - // Each sequence may be a different length, so we need to extract them - // manually. We will reshape them into a cube with each slice equal to - // a time step. - inputTemp = arma::cube(trainInput.at(0, j).memptr(), inputSize, 1, - trainInput.at(0, j).n_elem / inputSize, false, true); - labelsTemp = arma::cube(trainLabels.at(0, j).memptr(), inputSize, 1, - trainInput.at(0, j).n_elem / inputSize, false, true); - - model.Rho() = inputTemp.n_elem / inputSize; - model.Train(inputTemp, labelsTemp, opt); - opt.ResetPolicy() = false; - } - } - - double error = 0; - - // Ask the network to predict the next Reber grammar in the given sequence. - for (size_t i = 0; i < testReberGrammarCount; ++i) - { - arma::cube prediction; - arma::cube input(testInput.at(0, i).memptr(), inputSize, 1, - testInput.at(0, i).n_elem / inputSize, false, true); - - model.Rho() = input.n_elem / inputSize; - model.Predict(input, prediction); - - const size_t reberGrammerSize = 7; - std::string inputReber = ""; - - size_t reberError = 0; - - for (size_t j = 0; j < (prediction.n_elem / reberGrammerSize); ++j) - { - char predictedSymbol, inputSymbol; - std::string reberChoices; - - arma::umat output = (prediction.slice(j) == (arma::ones( - reberGrammerSize, 1) * - arma::as_scalar(arma::max(prediction.slice(j))))); - - ReberReverseTranslation(output, predictedSymbol); - ReberReverseTranslation(input.slice(j), inputSymbol); - inputReber += inputSymbol; - - if (recursive) - GenerateNextRecursiveReber(transitions, inputReber, reberChoices); - else - GenerateNextReber(transitions, inputReber, reberChoices); - - if (reberChoices.find(predictedSymbol) != std::string::npos) - reberError++; - } - - if (reberError != (prediction.n_elem / reberGrammerSize)) - error += 1; - } - - error /= testReberGrammarCount; - if (error <= 0.35) - { - ++successes; - break; - } - - offset += 3; - } - - REQUIRE(successes >= 1); -} - -/** - * Train the specified networks on an embedded Reber grammar dataset. - */ -TEST_CASE("CustomRecursiveReberGrammarTest", "[RecurrentNetworkTest]") -{ - ReberGrammarTestCustomNetwork(16, true); -} - -/** - * @brief Generates noisy sine wave and outputs the data and the labels that - * can be used directly for training and testing with RNN. - * - * @param data The data points as output - * @param labels The expected values as output - * @param rho The size of the sequence of each data point - * @param outputSteps How many output steps to consider for every rho inputs - * @param dataPoints The number of generated data points. The actual generated - * data points may be more than this to adjust to the outputSteps. But at - * the minimum these many data points will be generated. - * @param gain The gain on the amplitude - * @param freq The frquency of the sine wave - * @param phase The phase shift if any - * @param noisePercent The percent noise to induce - * @param numCycles How many full size wave cycles required. All the data - * points will be fit into these cycles. - * @param normalize Whether to normalise the data. This may be required for some - * layers like LSTM. Default is true. - */ -void GenerateNoisySinRNN(arma::cube& data, - arma::cube& labels, - size_t rho, - size_t outputSteps = 1, - const int dataPoints = 100, - const double gain = 1.0, - const int freq = 10, - const double phase = 0, - const int noisePercent = 20, - const double numCycles = 6.0, - const bool normalize = true) -{ - int points = dataPoints; - int r = dataPoints % rho; - - if (r == 0) - { - points += outputSteps; - } - else - { - points += rho - r + outputSteps; - } - - arma::colvec x(points); - int i = 0; - double interval = numCycles / freq / points; - - x.for_each([&i, gain, freq, phase, noisePercent, interval] - (arma::colvec::elem_type& val) { - double t = interval * (++i); - val = gain * ::sin(2 * M_PI * freq * t + phase) + - (noisePercent * gain / 100 * Random(0.0, 0.1)); - }); - - arma::colvec y = x; - if (normalize) - y = arma::normalise(x); - - // Now break this into columns of rho size slices. - size_t numColumns = y.n_elem / rho; - data = arma::cube(1, numColumns, rho); - labels = arma::cube(outputSteps, numColumns, 1); - - for (size_t i = 0; i < numColumns; ++i) - { - data.tube(0, i) = y.rows(i * rho, i * rho + rho - 1); - labels.subcube(0, i, 0, outputSteps - 1, i, 0) = - y.rows(i * rho + rho, i * rho + rho + outputSteps - 1); - } -} - -/** - * @brief RNNSineTest Test a simple RNN using noisy sine. Use single output - * for multiple inputs. - * @param hiddenUnits No of units in the hiddenlayer. - * @param rho The input sequence length. - * @param numEpochs The number of epochs to run. - * @return The mean squared error of the prediction. - */ -double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100) -{ - RNN > net(rho, true); - net.Add >(1, hiddenUnits); - net.Add >(hiddenUnits, hiddenUnits); - net.Add >(hiddenUnits, 1); - - RMSProp opt(0.005, 100, 0.9, 1e-08, 50000, 1e-5); - - // Generate data - arma::cube data; - arma::cube labels; - GenerateNoisySinRNN(data, labels, rho, 1, 2000, 20.0, 200, 0.0, 45, 20); - - // Break into training and test sets. Simply split along columns. - size_t trainCols = data.n_cols * 0.8; // Take 20% out for testing. - size_t testCols = data.n_cols - trainCols; - arma::cube testData = data.subcube(0, data.n_cols - testCols, 0, - data.n_rows - 1, data.n_cols - 1, data.n_slices - 1); - arma::cube testLabels = labels.subcube(0, labels.n_cols - testCols, 0, - labels.n_rows - 1, labels.n_cols - 1, labels.n_slices - 1); - - for (size_t i = 0; i < numEpochs; ++i) - { - net.Train(data.subcube(0, 0, 0, data.n_rows - 1, trainCols - 1, - data.n_slices - 1), labels.subcube(0, 0, 0, labels.n_rows - 1, - trainCols - 1, labels.n_slices - 1), opt); - } - // Well now it should be trained. Do the test here. - arma::cube prediction; - net.Predict(testData, prediction); - - // The prediction must really follow the test data. So convert both the test - // data and the pediction to vectors and compare the two. - arma::colvec testVector = arma::vectorise(testData); - arma::colvec predVector = arma::vectorise(prediction); - - // Adjust the vectors for comparison, as the prediction is one step ahead. - testVector = testVector.rows(1, testVector.n_rows - 1); - predVector = predVector.rows(0, predVector.n_rows - 2); - double error = std::sqrt(arma::sum(arma::square(testVector - predVector))) / - testVector.n_rows; - - return error; -} - -/** - * Test RNN using multiple timestep input and single output. - */ -TEST_CASE("MultiTimestepTest", "[RecurrentNetworkTest]") -{ - double err = RNNSineTest(4, 10, 20); - REQUIRE(err <= 0.025); -} - -/** - * Test that RNN::Train() returns finite objective value. - */ -TEST_CASE("RNNTrainReturnObjective", "[RecurrentNetworkTest]") -{ - const size_t rho = 10; - - // Generate 12 (2 * 6) noisy sines. A single sine contains rho - // points/features. - arma::cube input; - arma::mat labelsTemp; - GenerateNoisySines(input, labelsTemp, rho, 6); - - arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); - for (size_t i = 0; i < labelsTemp.n_cols; ++i) - { - const int value = arma::as_scalar(arma::find( - arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; - labels.tube(0, i).fill(value); - } - - /** - * Construct a network with 1 input unit, 4 hidden units and 10 output - * units. The hidden layer is connected to itself. The network structure - * looks like: - * - * Input Hidden Output - * Layer(1) Layer(4) Layer(10) - * +-----+ +-----+ +-----+ - * | | | | | | - * | +------>| +------>| | - * | | ..>| | | | - * +-----+ . +--+--+ +-----+ - * . . - * . . - * ....... - */ - Add<> add(4); - Linear<> lookup(1, 4); - SigmoidLayer<> sigmoidLayer; - Linear<> linear(4, 4); - Recurrent<>* recurrent = new Recurrent<>(add, lookup, linear, - sigmoidLayer, rho); - - RNN<> model(rho); - model.Add >(); - model.Add(recurrent); - model.Add >(4, 10); - model.Add >(); - - StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch */, -100); - double objVal = model.Train(input, labels, opt); - - REQUIRE(std::isfinite(objVal) == true); -} - -/** - * Test that BRNN::Train() returns finite objective value. - */ -TEST_CASE("BRNNTrainReturnObjective", "[RecurrentNetworkTest]") -{ - const size_t rho = 10; - - arma::cube input; - arma::mat labelsTemp; - GenerateNoisySines(input, labelsTemp, rho, 6); - - arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); - for (size_t i = 0; i < labelsTemp.n_cols; ++i) - { - const int value = arma::as_scalar(arma::find( - arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; - labels.tube(0, i).fill(value); - } - - Add<> add(4); - Linear<> lookup(1, 4); - SigmoidLayer<> sigmoidLayer; - Linear<> linear(4, 4); - Recurrent<>* recurrent = new Recurrent<>( - add, lookup, linear, sigmoidLayer, rho); - - BRNN<> model(rho); - model.Add >(); - model.Add(recurrent); - model.Add >(4, 5); - - StandardSGD opt(0.1, 1, 500 * input.n_cols, -100); - double objVal = model.Train(input, labels, opt); - INFO("Training over"); - - // Test that BRNN::Train() returns finite objective value. - REQUIRE(std::isfinite(objVal) == true); -} - -/** - * Test that RNN::Train() does not give an error for large rho. - */ -TEST_CASE("LargeRhoValueRnnTest", "[RecurrentNetworkTest]") -{ - // Setting rho value greater than sequence length which is 17. - const size_t rho = 100; - const size_t hiddenSize = 128; - const size_t numLetters = 256; - using MatType = arma::cube; - std::vectortrainingData = { "THIS IS THE INPUT 0" , - "THIS IS THE INPUT 1" , - "THIS IS THE INPUT 3"}; - - - RNN<> model(rho); - model.Add>(); - model.Add>(numLetters, hiddenSize, rho); - model.Add>(0.1); - model.Add>(hiddenSize, numLetters); - - const auto makeInput = [numLetters](const char *line) -> MatType - { - const auto strLen = strlen(line); - // Rows: number of dimensions. - // Cols: number of sequences/points. - // Slices: number of steps in sequences. - MatType result(numLetters, 1, strLen, arma::fill::zeros); - for (size_t i = 0; i < strLen; ++i) - { - result.at(static_cast(line[i]), 0, i) = 1.0; - } - return result; - }; - - const auto makeTarget = [] (const char *line) -> MatType - { - const auto strLen = strlen(line); - // Responses for NegativeLogLikelihood should be - // non-one-hot-encoded class IDs (from 1 to num_classes). - MatType result(1, 1, strLen, arma::fill::zeros); - // The response is the *next* letter in the sequence. - for (size_t i = 0; i < strLen - 1; ++i) - { - result.at(0, 0, i) = static_cast(line[i + 1]) + 1.0; - } - // The final response is empty, so we set it to class 0. - result.at(0, 0, strLen - 1) = 1.0; - return result; - }; - - std::vector inputs(trainingData.size()); - std::vector targets(trainingData.size()); - for (size_t i = 0; i < trainingData.size(); ++i) - { - inputs[i] = makeInput(trainingData[i].c_str()); - targets[i] = makeTarget(trainingData[i].c_str()); - } - ens::SGD<> opt(0.01, 1, 100); - model.Train(inputs[0], targets[0], opt); - INFO("Training over"); -}