From 34cf419bb86658f137ee2414879e4be9e514d9f3 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Sun, 31 Jan 2021 19:05:00 +0100 Subject: [PATCH] Update FFN tests to use the base layer class. --- src/mlpack/methods/ann/layer/add.hpp | 4 + src/mlpack/methods/ann/layer/base_layer.hpp | 1 + src/mlpack/methods/ann/layer/celu.hpp | 2 +- .../methods/ann/layer/dropconnect_impl.hpp | 2 + src/mlpack/methods/ann/layer/highway.hpp | 4 +- src/mlpack/methods/ann/layer/layer_types.hpp | 1 + src/mlpack/tests/CMakeLists.txt | 2 +- src/mlpack/tests/feedforward_network_test.cpp | 566 ++++++++++++++++++ 8 files changed, 578 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/ann/layer/add.hpp b/src/mlpack/methods/ann/layer/add.hpp index 4955b68503..d10cbfa049 100644 --- a/src/mlpack/methods/ann/layer/add.hpp +++ b/src/mlpack/methods/ann/layer/add.hpp @@ -13,6 +13,7 @@ #define MLPACK_METHODS_ANN_LAYER_ADD_HPP #include +#include "layer.hpp" #include namespace mlpack { @@ -41,6 +42,9 @@ class AddType : public Layer */ AddType(const size_t outSize = 0); + //! Clone the AddType object. This handles polymorphism correctly. + AddType* Clone() const { return new AddType(*this); } + /** * Ordinary feed forward pass of a neural network, evaluating the function * f(x) by propagating the activity forward through f. diff --git a/src/mlpack/methods/ann/layer/base_layer.hpp b/src/mlpack/methods/ann/layer/base_layer.hpp index 5c6efd2dbc..fe3ab26eb6 100644 --- a/src/mlpack/methods/ann/layer/base_layer.hpp +++ b/src/mlpack/methods/ann/layer/base_layer.hpp @@ -171,6 +171,7 @@ typedef BaseLayer Identity; // using ReLULayer = BaseLayer< // ActivationFunction, InputDataType, OutputDataType>; typedef BaseLayer ReLULayer; +typedef BaseLayer ReLU; /** * Standard hyperbolic tangent layer. diff --git a/src/mlpack/methods/ann/layer/celu.hpp b/src/mlpack/methods/ann/layer/celu.hpp index b86c42c345..c86afd8819 100644 --- a/src/mlpack/methods/ann/layer/celu.hpp +++ b/src/mlpack/methods/ann/layer/celu.hpp @@ -70,7 +70,7 @@ class CELUType : public Layer CELUType(const double alpha = 1.0); //! Clone the CELUType object. This handles polymorphism correctly. - CELUType* Clone() const { return new CELUType(*this); } + CELUType* Clone() const { return new CELUType(*this); } /** * Ordinary feed forward pass of a neural network, evaluating the function diff --git a/src/mlpack/methods/ann/layer/dropconnect_impl.hpp b/src/mlpack/methods/ann/layer/dropconnect_impl.hpp index ecca4fd10f..ae57d91231 100644 --- a/src/mlpack/methods/ann/layer/dropconnect_impl.hpp +++ b/src/mlpack/methods/ann/layer/dropconnect_impl.hpp @@ -17,6 +17,8 @@ // In case it hasn't yet been included. #include "dropconnect.hpp" +#include "linear.hpp" + namespace mlpack { namespace ann /** Artificial Neural Network. */ { diff --git a/src/mlpack/methods/ann/layer/highway.hpp b/src/mlpack/methods/ann/layer/highway.hpp index 4b92f4610c..7b0427bc6d 100644 --- a/src/mlpack/methods/ann/layer/highway.hpp +++ b/src/mlpack/methods/ann/layer/highway.hpp @@ -65,8 +65,8 @@ class HighwayType : public Layer //! Destroy the Highway object. ~HighwayType(); - //! Clone the HighwayType object. This handles polymorphism correctly. - HighwayType* Clone() const { return new HighwayType(*this); } + //! Clone the HighwayType object. This handles polymorphism correctly. + HighwayType* Clone() const { return new HighwayType(*this); } /** * Reset the layer parameter. diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 0a20e8c13b..d843bb986a 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 1c7879a709..e067d9848e 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -5,7 +5,7 @@ add_executable(mlpack_test # akfn_test.cpp # aknn_test.cpp # ann_dist_test.cpp - ann_layer_test.cpp + # ann_layer_test.cpp # ann_regularizer_test.cpp # ann_test_tools.hpp # ann_visitor_test.cpp diff --git a/src/mlpack/tests/feedforward_network_test.cpp b/src/mlpack/tests/feedforward_network_test.cpp index 8608ee343f..2a83380b56 100644 --- a/src/mlpack/tests/feedforward_network_test.cpp +++ b/src/mlpack/tests/feedforward_network_test.cpp @@ -337,3 +337,569 @@ TEST_CASE("CheckCopyMovingDropoutNetworkTest", "[FeedForwardNetworkTest]") // Check whether move constructor is working or not. CheckMoveFunction(model1, trainData, trainLabels, 1); } + +/** + * Train the vanilla network on a larger dataset. + */ +TEST_CASE("FFVanillaNetworkTest", "[FeedForwardNetworkTest]") +{ + // Load the dataset. + arma::mat trainData; + data::Load("thyroid_train.csv", trainData, true); + + arma::mat trainLabels = trainData.row(trainData.n_rows - 1); + trainData.shed_row(trainData.n_rows - 1); + + arma::mat testData; + data::Load("thyroid_test.csv", testData, true); + + arma::mat testLabels = testData.row(testData.n_rows - 1); + testData.shed_row(testData.n_rows - 1); + + /* + * Construct a feed forward network with trainData.n_rows input nodes, + * hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The + * network structure looks like: + * + * Input Hidden Output + * Layer Layer Layer + * +-----+ +-----+ +-----+ + * | | | | | | + * | +------>| +------>| | + * | | +>| | +>| | + * +-----+ | +--+--+ | +-----+ + * | | + * Bias | Bias | + * Layer | Layer | + * +-----+ | +-----+ | + * | | | | | | + * | +-----+ | +-----+ + * | | | | + * +-----+ +-----+ + */ + + FFN > model; + model.Add(trainData.n_rows, 8); + model.Add(); + model.Add(8, 3); + model.Add(); + + // Vanilla neural net with logistic activation function. + // Because 92% of the patients are not hyperthyroid the neural + // network must be significant better than 92%. + TestNetwork<>(model, trainData, trainLabels, testData, testLabels, 10, 0.1); + + arma::mat dataset; + dataset.load("mnist_first250_training_4s_and_9s.arm"); + + // Normalize each point since these are images. + for (size_t i = 0; i < dataset.n_cols; ++i) + dataset.col(i) /= norm(dataset.col(i), 2); + + arma::mat labels = arma::zeros(1, dataset.n_cols); + labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); + labels += 1; + + FFN > model1; + model1.Add(dataset.n_rows, 10); + model1.Add(); + model1.Add(10, 2); + model1.Add(); + // Vanilla neural net with logistic activation function. + TestNetwork(model1, dataset, labels, dataset, labels, 10, 0.2); +} + +TEST_CASE("ForwardBackwardTest", "[FeedForwardNetworkTest]") +{ + arma::mat dataset; + dataset.load("mnist_first250_training_4s_and_9s.arm"); + + // Normalize each point since these are images. + for (size_t i = 0; i < dataset.n_cols; ++i) + dataset.col(i) /= norm(dataset.col(i), 2); + + arma::mat labels = arma::zeros(1, dataset.n_cols); + labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); + labels += 1; + + FFN > model; + model.Add(dataset.n_rows, 50); + model.Add(); + model.Add(50, 10); + model.Add(); + + ens::VanillaUpdate opt; + model.ResetParameters(); + #if ENS_VERSION_MAJOR == 1 + opt.Initialize(model.Parameters().n_rows, model.Parameters().n_cols); + #else + ens::VanillaUpdate::Policy optPolicy(opt, + model.Parameters().n_rows, model.Parameters().n_cols); + #endif + double stepSize = 0.01; + size_t batchSize = 10; + + size_t iteration = 0; + bool converged = false; + while (iteration < 100) + { + arma::running_stat error; + size_t batchStart = 0; + while (batchStart < dataset.n_cols) + { + size_t batchEnd = std::min(batchStart + batchSize, + (size_t) dataset.n_cols); + arma::mat currentData = dataset.cols(batchStart, batchEnd - 1); + arma::mat currentLabels = labels.cols(batchStart, batchEnd - 1); + arma::mat currentResuls; + model.Forward(currentData, currentResuls); + arma::mat gradients; + model.Backward(currentData, currentLabels, gradients); + #if ENS_VERSION_MAJOR == 1 + opt.Update(model.Parameters(), stepSize, gradients); + #else + optPolicy.Update(model.Parameters(), stepSize, gradients); + #endif + batchStart = batchEnd; + + arma::mat prediction = arma::zeros(1, currentResuls.n_cols); + + for (size_t i = 0; i < currentResuls.n_cols; ++i) + { + prediction(i) = arma::as_scalar(arma::find( + arma::max(currentResuls.col(i)) == currentResuls.col(i), 1)) + 1; + } + + size_t correct = arma::accu(prediction == currentLabels); + error(1 - (double) correct / batchSize); + } + Log::Debug << "Current training error: " << error.mean() << std::endl; + iteration++; + if (error.mean() < 0.05) + { + converged = true; + break; + } + } + + REQUIRE(converged); +} + +/** + * Train the dropout network on a larger dataset. + */ +TEST_CASE("DropoutNetworkTest", "[FeedForwardNetworkTest]") +{ + // Load the dataset. + arma::mat trainData; + data::Load("thyroid_train.csv", trainData, true); + + arma::mat trainLabels = trainData.row(trainData.n_rows - 1); + trainData.shed_row(trainData.n_rows - 1); + + arma::mat testData; + data::Load("thyroid_test.csv", testData, true); + + arma::mat testLabels = testData.row(testData.n_rows - 1); + testData.shed_row(testData.n_rows - 1); + + /* + * Construct a feed forward network with trainData.n_rows input nodes, + * hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The + * network structure looks like: + * + * Input Hidden Dropout Output + * Layer Layer Layer Layer + * +-----+ +-----+ +-----+ +-----+ + * | | | | | | | | + * | +------>| +------>| +------>| | + * | | +>| | | | | | + * +-----+ | +--+--+ +-----+ +-----+ + * | + * Bias | + * Layer | + * +-----+ | + * | | | + * | +-----+ + * | | + * +-----+ + */ + + FFN > model; + model.Add(trainData.n_rows, 8); + model.Add(); + model.Add(); + model.Add(8, 3); + model.Add(); + + // Vanilla neural net with logistic activation function. + // Because 92% of the patients are not hyperthyroid the neural + // network must be significant better than 92%. + TestNetwork<>(model, trainData, trainLabels, testData, testLabels, 10, 0.1); + arma::mat dataset; + dataset.load("mnist_first250_training_4s_and_9s.arm"); + + // Normalize each point since these are images. + for (size_t i = 0; i < dataset.n_cols; ++i) + { + dataset.col(i) /= norm(dataset.col(i), 2); + } + + arma::mat labels = arma::zeros(1, dataset.n_cols); + labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); + labels += 1; + + FFN > model1; + model1.Add(dataset.n_rows, 10); + model1.Add(); + model.Add(); + model1.Add(10, 2); + model1.Add(); + // Vanilla neural net with logistic activation function. + TestNetwork(model1, dataset, labels, dataset, labels, 10, 0.2); +} + +/** + * Train the highway network on a larger dataset. + */ +TEST_CASE("HighwayNetworkTest", "[FeedForwardNetworkTest]") +{ + arma::mat dataset; + dataset.load("mnist_first250_training_4s_and_9s.arm"); + + // Normalize each point since these are images. + for (size_t i = 0; i < dataset.n_cols; ++i) + dataset.col(i) /= norm(dataset.col(i), 2); + + arma::mat labels = arma::zeros(1, dataset.n_cols); + labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); + labels += 1; + + FFN > model; + model.Add(dataset.n_rows, 10); + Highway* highway = new Highway(10, true); + highway->Add(10, 10); + highway->Add(); + model.Add(highway); // This takes ownership of the memory. + model.Add(10, 2); + model.Add(); + TestNetwork(model, dataset, labels, dataset, labels, 10, 0.2); +} + +/** + * Train the DropConnect network on a larger dataset. + */ +TEST_CASE("DropConnectNetworkTest", "[FeedForwardNetworkTest]") +{ + // Load the dataset. + arma::mat trainData; + data::Load("thyroid_train.csv", trainData, true); + + arma::mat trainLabels = trainData.row(trainData.n_rows - 1); + trainData.shed_row(trainData.n_rows - 1); + + arma::mat testData; + data::Load("thyroid_test.csv", testData, true); + + arma::mat testLabels = testData.row(testData.n_rows - 1); + testData.shed_row(testData.n_rows - 1); + + /* + * Construct a feed forward network with trainData.n_rows input nodes, + * hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The + * network struct that looks like: + * + * Input Hidden DropConnect Output + * Layer Layer Layer Layer + * +-----+ +-----+ +-----+ +-----+ + * | | | | | | | | + * | +------>| +------>| +------>| | + * | | +>| | | | | | + * +-----+ | +--+--+ +-----+ +-----+ + * | + * Bias | + * Layer | + * +-----+ | + * | | | + * | +-----+ + * | | + * +-----+ + * + * + */ + + FFN > model; + model.Add(trainData.n_rows, 8); + model.Add(); + model.Add(8, 3); + model.Add(); + + // Vanilla neural net with logistic activation function. + // Because 92% of the patients are not hyperthyroid the neural + // network must be significant better than 92%. + TestNetwork(model, trainData, trainLabels, testData, testLabels, 10, 0.1); + + arma::mat dataset; + dataset.load("mnist_first250_training_4s_and_9s.arm"); + + // Normalize each point since these are images. + for (size_t i = 0; i < dataset.n_cols; ++i) + dataset.col(i) /= norm(dataset.col(i), 2); + + arma::mat labels = arma::zeros(1, dataset.n_cols); + labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); + labels += 1; + + FFN > model1; + model1.Add(dataset.n_rows, 10); + model1.Add(); + model1.Add(10, 2); + model1.Add(); + + // Vanilla neural net with logistic activation function. + TestNetwork(model1, dataset, labels, dataset, labels, 10, 0.2); +} + +/** + * Test miscellaneous things of FFN, + * e.g. copy/move constructor, assignment operator. + */ +TEST_CASE("FFNMiscTest", "[FeedForwardNetworkTest]") +{ + FFN> model; + model.Add(2, 3); + model.Add(); + + auto copiedModel(model); + copiedModel = model; + auto movedModel(std::move(model)); + auto moveOperator = std::move(copiedModel); +} + +/** + * Test that serialization works ok. + */ +TEST_CASE("FFSerializationTest", "[FeedForwardNetworkTest]") +{ + // Load the dataset. + arma::mat trainData; + data::Load("thyroid_train.csv", trainData, true); + + arma::mat trainLabels = trainData.row(trainData.n_rows - 1); + trainData.shed_row(trainData.n_rows - 1); + + arma::mat testData; + data::Load("thyroid_test.csv", testData, true); + + arma::mat testLabels = testData.row(testData.n_rows - 1); + testData.shed_row(testData.n_rows - 1); + + // Vanilla neural net with logistic activation function. + // Because 92% of the patients are not hyperthyroid the neural + // network must be significant better than 92%. + FFN > model; + model.Add(trainData.n_rows, 8); + model.Add(); + model.Add(); + model.Add(8, 3); + model.Add(); + + ens::RMSProp opt(0.01, 32, 0.88, 1e-8, trainData.n_cols /* 1 epoch */, -1); + + model.Train(trainData, trainLabels, opt); + + FFN> xmlModel, jsonModel, binaryModel; + xmlModel.Add(10, 10); // Layer that will get removed. + + // Serialize into other models. + SerializeObjectAll(model, xmlModel, jsonModel, binaryModel); + + arma::mat predictions, xmlPredictions, jsonPredictions, binaryPredictions; + model.Predict(testData, predictions); + xmlModel.Predict(testData, xmlPredictions); + jsonModel.Predict(testData, jsonPredictions); + jsonModel.Predict(testData, binaryPredictions); + + // TODO: serialization + //CheckMatrices(predictions, xmlPredictions, jsonPredictions, + // binaryPredictions); +} + +/** + * Test the overload of Forward function which allows partial forward pass. + */ +TEST_CASE("PartialForwardTest", "[FeedForwardNetworkTest]") +{ + FFN, RandomInitialization> model; + model.Add(5, 10); + + // Add a new Add<> module which adds a constant term to the input. + Add* addModule = new Add(10); + model.Add(addModule); + + LinearNoBias* linearNoBiasModule = new LinearNoBias(10, 10); + model.Add(linearNoBiasModule); + + model.Add(10, 10); + + model.ResetParameters(); + // Set the parameters of the Add<> module to a matrix of ones. + addModule->Parameters() = arma::ones(10, 1); + // Set the parameters of the LinearNoBias<> module to a matrix of ones. + linearNoBiasModule->Parameters() = arma::ones(10, 10); + + arma::mat input = arma::ones(10, 1); + arma::mat output; + + // Forward pass only through the Add module. + model.Forward(input, + output, + 1 /* Index of the Add module */, + 1 /* Index of the Add module */); + + // As we only forward pass through Add module, input and output should + // differ by a matrix of ones. + CheckMatrices(input, output - 1); + + // Forward pass only through the Add module and the LinearNoBias module. + model.Forward(input, + output, + 1 /* Index of the Add module */, + 2 /* Index of the LinearNoBias module */); + + // As we only forward pass through Add module followed by the LinearNoBias + // module, output should be a matrix of 20s.(output = weight * input) + CheckMatrices(output, arma::ones(10, 1) * 20); +} + +/** + * Test that FFN::Train() returns finite objective value. + */ +TEST_CASE("FFNTrainReturnObjective", "[FeedForwardNetworkTest]") +{ + // Load the dataset. + arma::mat trainData; + data::Load("thyroid_train.csv", trainData, true); + + arma::mat trainLabels = trainData.row(trainData.n_rows - 1); + trainData.shed_row(trainData.n_rows - 1); + + arma::mat testData; + data::Load("thyroid_test.csv", testData, true); + + arma::mat testLabels = testData.row(testData.n_rows - 1); + testData.shed_row(testData.n_rows - 1); + + // Vanilla neural net with logistic activation function. + // Because 92% of the patients are not hyperthyroid the neural + // network must be significantly better than 92%. + FFN > model; + model.Add(trainData.n_rows, 8); + model.Add(); + model.Add(); + model.Add(8, 3); + model.Add(); + + ens::RMSProp opt(0.01, 32, 0.88, 1e-8, trainData.n_cols /* 1 epoch */, -1); + + double objVal = model.Train(trainData, trainLabels, opt); + + REQUIRE(std::isfinite(objVal) == true); +} + +/** + * Test that FFN::Model() allows us to access the instantiated network. + */ +TEST_CASE("FFNReturnModel", "[FeedForwardNetworkTest]") +{ + // Create dummy network. + FFN > model; + Linear* linearA = new Linear(3, 3); + model.Add(linearA); + Linear* linearB = new Linear(3, 4); + model.Add(linearB); + + // Initialize network parameter. + model.ResetParameters(); + + // Set all network parameter to one. + model.Parameters().ones(); + + // Zero the second layer parameter. + linearB->Parameters().zeros(); + + // Get the layer parameter from layer A and layer B and store them in + // parameterA and parameterB. + const arma::mat parameterA = model.Model()[0]->Parameters(); + const arma::mat parameterB = model.Model()[1]->Parameters(); + + CheckMatrices(parameterA, arma::ones(3 * 3 + 3, 1)); + CheckMatrices(parameterB, arma::zeros(3 * 4 + 4, 1)); + + CheckMatrices(linearA->Parameters(), arma::ones(3 * 3 + 3, 1)); + CheckMatrices(linearB->Parameters(), arma::zeros(3 * 4 + 4, 1)); +} + +/** + * Test to see if the FFN code compiles when the Optimizer + * doesn't have the MaxIterations() method. + */ +TEST_CASE("OptimizerTest", "[FeedForwardNetworkTest]") +{ + // Load the dataset. + arma::mat trainData; + data::Load("thyroid_train.csv", trainData, true); + + arma::mat trainLabels = trainData.row(trainData.n_rows - 1); + trainData.shed_row(trainData.n_rows - 1); + + arma::mat testData; + data::Load("thyroid_test.csv", testData, true); + + arma::mat testLabels = testData.row(testData.n_rows - 1); + testData.shed_row(testData.n_rows - 1); + + FFN, RandomInitialization> model; + model.Add(trainData.n_rows, 8); + model.Add(8, 3); + model.Add(); + + ens::DE opt(200, 1000, 0.6, 0.8, 1e-5); + model.Train(trainData, trainLabels, opt); +} + +/** + * Test to see if an exception is thrown when input with + * wrong shape is provided to a FFN. + */ +TEST_CASE("FFNCheckInputShapeTest", "[FeedForwardNetworkTest]") +{ + // Load the dataset. + arma::mat trainData; + data::Load("thyroid_train.csv", trainData, true); + + arma::mat trainLabels = trainData.row(trainData.n_rows - 1); + trainData.shed_row(trainData.n_rows - 1); + + arma::mat testData; + data::Load("thyroid_test.csv", testData, true); + + arma::mat testLabels = testData.row(testData.n_rows - 1); + testData.shed_row(testData.n_rows - 1); + + FFN, RandomInitialization> model; + // Purposely putting wrong input shape so that error is thrown. + model.Add(trainData.n_rows - 3, 8); + model.Add(8, 3); + model.Add(); + + std::string expectedMsg = "FFN<>::Train(): "; + expectedMsg += "the first layer of the network expects "; + expectedMsg += std::to_string(trainData.n_rows - 3) + " elements, "; + expectedMsg += "but the input has " + std::to_string(trainData.n_rows) + + " dimensions! "; + + ens::DE opt(200, 1000, 0.6, 0.8, 1e-5); + + REQUIRE_THROWS_AS(model.Train(trainData, trainLabels, opt), std::logic_error); +}