Anmolpreet Singh
f29baecfbf
I have changed the test network with stochastic option to be off so that randomization was leading to different predictions.
981 lines
31 KiB
C++
981 lines
31 KiB
C++
/**
|
|
* @file tests/feedforward_network_test.cpp
|
|
* @author Marcus Edel
|
|
* @author Palash Ahuja
|
|
*
|
|
* Tests the feed forward network.
|
|
*
|
|
* mlpack is free software; you may redistribute it and/or modify it under the
|
|
* terms of the 3-clause BSD license. You should have received a copy of the
|
|
* 3-clause BSD license along with mlpack. If not, see
|
|
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
|
*/
|
|
#include <mlpack/core.hpp>
|
|
|
|
#include <mlpack/methods/ann/layer/layer.hpp>
|
|
#include <mlpack/methods/ann/loss_functions/mean_squared_error.hpp>
|
|
#include <mlpack/methods/ann/ffn.hpp>
|
|
|
|
#include <ensmallen.hpp>
|
|
|
|
#include "catch.hpp"
|
|
#include "serialization.hpp"
|
|
#include "custom_layer.hpp"
|
|
|
|
using namespace mlpack;
|
|
using namespace mlpack::ann;
|
|
|
|
/**
|
|
* Train and evaluate a model with the specified structure.
|
|
*/
|
|
template<typename MatType = arma::mat, typename ModelType>
|
|
void TestNetwork(ModelType& model,
|
|
MatType& trainData,
|
|
MatType& trainLabels,
|
|
MatType& testData,
|
|
MatType& testLabels,
|
|
const size_t maxEpochs,
|
|
const double classificationErrorThreshold)
|
|
{
|
|
ens::RMSProp opt(0.01, 32, 0.88, 1e-8, maxEpochs * trainData.n_cols, -1);
|
|
model.Train(trainData, trainLabels, opt);
|
|
|
|
MatType predictionTemp;
|
|
model.Predict(testData, predictionTemp);
|
|
MatType prediction = arma::zeros<MatType>(1, predictionTemp.n_cols);
|
|
|
|
for (size_t i = 0; i < predictionTemp.n_cols; ++i)
|
|
{
|
|
prediction(i) = arma::as_scalar(arma::find(
|
|
arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1;
|
|
}
|
|
|
|
size_t correct = arma::accu(prediction == testLabels);
|
|
double classificationError = 1 - double(correct) / testData.n_cols;
|
|
REQUIRE(classificationError <= classificationErrorThreshold);
|
|
}
|
|
|
|
// network1 should be allocated with `new`, and trained on some data.
|
|
template<typename MatType = arma::mat, typename ModelType>
|
|
void CheckCopyFunction(ModelType* network1,
|
|
MatType& trainData,
|
|
MatType& trainLabels,
|
|
const size_t maxEpochs)
|
|
{
|
|
ens::RMSProp opt(0.01, 32, 0.88, 1e-8, maxEpochs * trainData.n_cols, -1);
|
|
network1->Train(trainData, trainLabels, opt);
|
|
|
|
arma::mat predictions1;
|
|
network1->Predict(trainData, predictions1);
|
|
FFN<> network2;
|
|
network2 = *network1;
|
|
delete network1;
|
|
|
|
// Deallocating all of network1's memory, so that network2 does not use any
|
|
// of that memory.
|
|
arma::mat predictions2;
|
|
network2.Predict(trainData, predictions2);
|
|
CheckMatrices(predictions1, predictions2);
|
|
}
|
|
|
|
// network1 should be allocated with `new`, and trained on some data.
|
|
template<typename MatType = arma::mat, typename ModelType>
|
|
void CheckMoveFunction(ModelType* network1,
|
|
MatType& trainData,
|
|
MatType& trainLabels,
|
|
const size_t maxEpochs)
|
|
{
|
|
ens::RMSProp opt(0.01, 32, 0.88, 1e-8, maxEpochs * trainData.n_cols, -1);
|
|
network1->Train(trainData, trainLabels, opt);
|
|
|
|
arma::mat predictions1;
|
|
network1->Predict(trainData, predictions1);
|
|
FFN<> network2(std::move(*network1));
|
|
delete network1;
|
|
|
|
// Deallocating all of network1's memory, so that network2 does not use any
|
|
// of that memory.
|
|
arma::mat predictions2;
|
|
network2.Predict(trainData, predictions2);
|
|
CheckMatrices(predictions1, predictions2);
|
|
}
|
|
|
|
/**
|
|
* Check whether copying and moving Vanila network is working or not.
|
|
*/
|
|
TEST_CASE("CheckCopyMovingVanillaNetworkTest", "[FeedForwardNetworkTest]")
|
|
{
|
|
// Load the dataset.
|
|
arma::mat trainData;
|
|
data::Load("thyroid_train.csv", trainData, true);
|
|
|
|
arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
|
|
trainData.shed_row(trainData.n_rows - 1);
|
|
|
|
/*
|
|
* Construct a feed forward network with trainData.n_rows input nodes,
|
|
* hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The
|
|
* network structure looks like:
|
|
*
|
|
* Input Hidden Output
|
|
* Layer Layer Layer
|
|
* +-----+ +-----+ +-----+
|
|
* | | | | | |
|
|
* | +------>| +------>| |
|
|
* | | +>| | +>| |
|
|
* +-----+ | +--+--+ | +-----+
|
|
* | |
|
|
* Bias | Bias |
|
|
* Layer | Layer |
|
|
* +-----+ | +-----+ |
|
|
* | | | | | |
|
|
* | +-----+ | +-----+
|
|
* | | | |
|
|
* +-----+ +-----+
|
|
*/
|
|
|
|
FFN<NegativeLogLikelihood<> > *model = new FFN<NegativeLogLikelihood<> >;
|
|
model->Add<Linear<> >(trainData.n_rows, 8);
|
|
model->Add<SigmoidLayer<> >();
|
|
model->Add<Linear<> >(8, 3);
|
|
model->Add<LogSoftMax<> >();
|
|
|
|
FFN<NegativeLogLikelihood<> > *model1 = new FFN<NegativeLogLikelihood<> >;
|
|
model1->Add<Linear<> >(trainData.n_rows, 8);
|
|
model1->Add<SigmoidLayer<> >();
|
|
model1->Add<Linear<> >(8, 3);
|
|
model1->Add<LogSoftMax<> >();
|
|
|
|
// Check whether copy constructor is working or not.
|
|
CheckCopyFunction<>(model, trainData, trainLabels, 1);
|
|
|
|
// Check whether move constructor is working or not.
|
|
CheckMoveFunction<>(model1, trainData, trainLabels, 1);
|
|
}
|
|
|
|
/**
|
|
* Check whether copying and moving network with Reparametrization is working or not.
|
|
*/
|
|
TEST_CASE("CheckCopyMovingReparametrizationNetworkTest", "[FeedForwardNetworkTest]")
|
|
{
|
|
// Load the dataset.
|
|
arma::mat trainData;
|
|
data::Load("thyroid_train.csv", trainData, true);
|
|
|
|
arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
|
|
trainData.shed_row(trainData.n_rows - 1);
|
|
|
|
/*
|
|
* Construct a feed forward network with trainData.n_rows input nodes,
|
|
* followed by a linear layer and then a reparametrization layer
|
|
*/
|
|
|
|
FFN<NegativeLogLikelihood<> > *model = new FFN<NegativeLogLikelihood<> >;
|
|
model1->Add<Linear<> >(trainData.n_rows, 8);
|
|
model1->Add<Reparametrization<> >(4,false,true,1);
|
|
model->Add<LogSoftMax<> >();
|
|
|
|
FFN<NegativeLogLikelihood<> > *model1 = new FFN<NegativeLogLikelihood<> >;
|
|
model1->Add<Linear<> >(trainData.n_rows, 8);
|
|
model1->Add<Reparametrization<> >(4,false,true,1);
|
|
model1->Add<LogSoftMax<> >();
|
|
|
|
// Check whether copy constructor is working or not.
|
|
CheckCopyFunction<>(model, trainData, trainLabels, 1);
|
|
|
|
// Check whether move constructor is working or not.
|
|
CheckMoveFunction<>(model1, trainData, trainLabels, 1);
|
|
}
|
|
|
|
/**
|
|
* Check whether copying and moving network with linear3d is working or not.
|
|
*/
|
|
TEST_CASE("CheckCopyMovingLinear3DNetworkTest", "[FeedForwardNetworkTest]")
|
|
{
|
|
// Load the dataset.
|
|
arma::mat trainData;
|
|
data::Load("thyroid_train.csv", trainData, true);
|
|
|
|
arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
|
|
trainData.shed_row(trainData.n_rows - 1);
|
|
|
|
/*
|
|
* Construct a feed forward network with trainData.n_rows input nodes,
|
|
* hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The
|
|
* network structure looks like:
|
|
*
|
|
* Input Hidden Output
|
|
* Layer Layer Layer
|
|
* +-----+ +-----+ +-----+
|
|
* | | | | | |
|
|
* | +------>| +------>| |
|
|
* | | +>| | +>| |
|
|
* +-----+ | +--+--+ | +-----+
|
|
* | |
|
|
* Bias | Bias |
|
|
* Layer | Layer |
|
|
* +-----+ | +-----+ |
|
|
* | | | | | |
|
|
* | +-----+ | +-----+
|
|
* | | | |
|
|
* +-----+ +-----+
|
|
*/
|
|
|
|
FFN<NegativeLogLikelihood<> > *model = new FFN<NegativeLogLikelihood<> >;
|
|
model->Add<Linear<> >(trainData.n_rows, 8);
|
|
model->Add<SigmoidLayer<> >();
|
|
model->Add<Linear3D<> >(8, 3);
|
|
model->Add<LogSoftMax<> >();
|
|
|
|
FFN<NegativeLogLikelihood<> > *model1 = new FFN<NegativeLogLikelihood<> >;
|
|
model1->Add<Linear<> >(trainData.n_rows, 8);
|
|
model1->Add<SigmoidLayer<> >();
|
|
model1->Add<Linear3D<> >(8, 3);
|
|
model1->Add<LogSoftMax<> >();
|
|
|
|
// Check whether copy constructor is working or not.
|
|
CheckCopyFunction<>(model, trainData, trainLabels, 1);
|
|
|
|
// Check whether move constructor is working or not.
|
|
CheckMoveFunction<>(model1, trainData, trainLabels, 1);
|
|
}
|
|
|
|
/**
|
|
* Check whether copying and moving of Noisy Linear layer is working or not.
|
|
*/
|
|
TEST_CASE("CheckCopyMovingNoisyLinearTest", "[FeedForwardNetworkTest]")
|
|
{
|
|
// Create training input by 5x5 matrix.
|
|
arma::mat input = arma::randu(10,1);
|
|
// Create training output by 1 matrix.
|
|
arma::mat output = arma::mat("1");
|
|
|
|
// Check copying constructor.
|
|
FFN<NegativeLogLikelihood<>> *model1 = new FFN<NegativeLogLikelihood<>>();
|
|
model1->Predictors() = input;
|
|
model1->Responses() = output;
|
|
model1->Add<IdentityLayer<>>();
|
|
model1->Add<NoisyLinear<>>(10, 5);
|
|
model1->Add<Linear<> >(5, 1);
|
|
model1->Add<LogSoftMax<>>();
|
|
|
|
// Check whether copy constructor is working or not.
|
|
CheckCopyFunction<>(model1, input, output, 1);
|
|
|
|
// Check moving constructor.
|
|
FFN<NegativeLogLikelihood<>> *model2 = new FFN<NegativeLogLikelihood<>>();
|
|
model2->Predictors() = input;
|
|
model2->Responses() = output;
|
|
model2->Add<IdentityLayer<>>();
|
|
model2->Add<NoisyLinear<>>(10, 5);
|
|
model2->Add<Linear<> >(5, 1);
|
|
model2->Add<LogSoftMax<>>();
|
|
|
|
// Check whether move constructor is working or not.
|
|
CheckMoveFunction<>(model2, input, output, 1);
|
|
}
|
|
|
|
/**
|
|
* Check whether copying and moving of concatenate layer is working or not.
|
|
*/
|
|
TEST_CASE("CheckCopyMovingConcatenateTest", "[FeedForwardNetworkTest]")
|
|
{
|
|
// Create training input by 5x5 matrix.
|
|
arma::mat input = arma::randu(10,1);
|
|
// Create training output by 1 matrix.
|
|
arma::mat output = arma::mat("1");
|
|
|
|
// Check copying constructor.
|
|
FFN<NegativeLogLikelihood<>> *model1 = new FFN<NegativeLogLikelihood<>>();
|
|
model1->Predictors() = input;
|
|
model1->Responses() = output;
|
|
model1->Add<IdentityLayer<>>();
|
|
model1->Add<Linear<>>(10, 5);
|
|
|
|
// Create concatenate layer.
|
|
arma::mat concatMatrix = arma::ones(5, 1);
|
|
Concatenate<>* concatLayer = new Concatenate<>();
|
|
concatLayer->Concat() = concatMatrix;
|
|
|
|
// Add concatenate layer to the current network.
|
|
model1->Add(concatLayer);
|
|
model1->Add<Linear<> >(10, 5);
|
|
model1->Add<LogSoftMax<>>();
|
|
|
|
// Check whether copy constructor is working or not.
|
|
CheckCopyFunction<>(model1, input, output, 1);
|
|
|
|
// Check moving constructor.
|
|
FFN<NegativeLogLikelihood<>> *model2 = new FFN<NegativeLogLikelihood<>>();
|
|
model2->Predictors() = input;
|
|
model2->Responses() = output;
|
|
model2->Add<IdentityLayer<>>();
|
|
model2->Add<Linear<>>(10, 5);
|
|
|
|
// Create new concat layer.
|
|
Concatenate<>* concatLayer2 = new Concatenate<>();
|
|
concatLayer2->Concat() = concatMatrix;
|
|
|
|
// Add concatenate layer to the current network.
|
|
model2->Add(concatLayer2);
|
|
model2->Add<Linear<> >(10, 5);
|
|
model2->Add<LogSoftMax<>>();
|
|
|
|
// Check whether move constructor is working or not.
|
|
CheckMoveFunction<>(model2, input, output, 1);
|
|
}
|
|
|
|
/**
|
|
* Check whether copying and moving of Dropout network is working or not.
|
|
*/
|
|
TEST_CASE("CheckCopyMovingDropoutNetworkTest", "[FeedForwardNetworkTest]")
|
|
{
|
|
// Load the dataset.
|
|
arma::mat trainData;
|
|
data::Load("thyroid_train.csv", trainData, true);
|
|
|
|
arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
|
|
trainData.shed_row(trainData.n_rows - 1);
|
|
|
|
/*
|
|
* Construct a feed forward network with trainData.n_rows input nodes,
|
|
* hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The
|
|
* network structure looks like:
|
|
*
|
|
* Input Hidden Output
|
|
* Layer Layer Layer
|
|
* +-----+ +-----+ +-----+
|
|
* | | | | | |
|
|
* | +------>| +------>| |
|
|
* | | +>| | +>| |
|
|
* +-----+ | +--+--+ | +-----+
|
|
* | |
|
|
* Bias | Bias |
|
|
* Layer | Layer |
|
|
* +-----+ | +-----+ |
|
|
* | | | | | |
|
|
* | +-----+ | +-----+
|
|
* | | | |
|
|
* +-----+ +-----+
|
|
*/
|
|
|
|
FFN<NegativeLogLikelihood<> > *model = new FFN<NegativeLogLikelihood<> >;
|
|
model->Add<Linear<> >(trainData.n_rows, 8);
|
|
model->Add<SigmoidLayer<> >();
|
|
model->Add<Dropout<> >(0.3);
|
|
model->Add<Linear<> >(8, 3);
|
|
model->Add<LogSoftMax<> >();
|
|
|
|
FFN<NegativeLogLikelihood<> > *model1 = new FFN<NegativeLogLikelihood<> >;
|
|
model1->Add<Linear<> >(trainData.n_rows, 8);
|
|
model1->Add<SigmoidLayer<> >();
|
|
model1->Add<Dropout<> >(0.3);
|
|
model1->Add<Linear<> >(8, 3);
|
|
model1->Add<LogSoftMax<> >();
|
|
|
|
// Check whether copy constructor is working or not.
|
|
CheckCopyFunction<>(model, trainData, trainLabels, 1);
|
|
|
|
// Check whether move constructor is working or not.
|
|
CheckMoveFunction<>(model1, trainData, trainLabels, 1);
|
|
}
|
|
|
|
/**
|
|
* Train the vanilla network on a larger dataset.
|
|
*/
|
|
TEST_CASE("FFVanillaNetworkTest", "[FeedForwardNetworkTest]")
|
|
{
|
|
// Load the dataset.
|
|
arma::mat trainData;
|
|
data::Load("thyroid_train.csv", trainData, true);
|
|
|
|
arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
|
|
trainData.shed_row(trainData.n_rows - 1);
|
|
|
|
arma::mat testData;
|
|
data::Load("thyroid_test.csv", testData, true);
|
|
|
|
arma::mat testLabels = testData.row(testData.n_rows - 1);
|
|
testData.shed_row(testData.n_rows - 1);
|
|
|
|
/*
|
|
* Construct a feed forward network with trainData.n_rows input nodes,
|
|
* hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The
|
|
* network structure looks like:
|
|
*
|
|
* Input Hidden Output
|
|
* Layer Layer Layer
|
|
* +-----+ +-----+ +-----+
|
|
* | | | | | |
|
|
* | +------>| +------>| |
|
|
* | | +>| | +>| |
|
|
* +-----+ | +--+--+ | +-----+
|
|
* | |
|
|
* Bias | Bias |
|
|
* Layer | Layer |
|
|
* +-----+ | +-----+ |
|
|
* | | | | | |
|
|
* | +-----+ | +-----+
|
|
* | | | |
|
|
* +-----+ +-----+
|
|
*/
|
|
|
|
FFN<NegativeLogLikelihood<> > model;
|
|
model.Add<Linear<> >(trainData.n_rows, 8);
|
|
model.Add<SigmoidLayer<> >();
|
|
model.Add<Linear<> >(8, 3);
|
|
model.Add<LogSoftMax<> >();
|
|
|
|
// Vanilla neural net with logistic activation function.
|
|
// Because 92% of the patients are not hyperthyroid the neural
|
|
// network must be significant better than 92%.
|
|
TestNetwork<>(model, trainData, trainLabels, testData, testLabels, 10, 0.1);
|
|
|
|
arma::mat dataset;
|
|
dataset.load("mnist_first250_training_4s_and_9s.arm");
|
|
|
|
// Normalize each point since these are images.
|
|
for (size_t i = 0; i < dataset.n_cols; ++i)
|
|
dataset.col(i) /= norm(dataset.col(i), 2);
|
|
|
|
arma::mat labels = arma::zeros(1, dataset.n_cols);
|
|
labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1);
|
|
labels += 1;
|
|
|
|
FFN<NegativeLogLikelihood<> > model1;
|
|
model1.Add<Linear<> >(dataset.n_rows, 10);
|
|
model1.Add<SigmoidLayer<> >();
|
|
model1.Add<Linear<> >(10, 2);
|
|
model1.Add<LogSoftMax<> >();
|
|
// Vanilla neural net with logistic activation function.
|
|
TestNetwork<>(model1, dataset, labels, dataset, labels, 10, 0.2);
|
|
}
|
|
|
|
TEST_CASE("ForwardBackwardTest", "[FeedForwardNetworkTest]")
|
|
{
|
|
arma::mat dataset;
|
|
dataset.load("mnist_first250_training_4s_and_9s.arm");
|
|
|
|
// Normalize each point since these are images.
|
|
for (size_t i = 0; i < dataset.n_cols; ++i)
|
|
dataset.col(i) /= norm(dataset.col(i), 2);
|
|
|
|
arma::mat labels = arma::zeros(1, dataset.n_cols);
|
|
labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1);
|
|
labels += 1;
|
|
|
|
FFN<NegativeLogLikelihood<> > model;
|
|
model.Add<Linear<> >(dataset.n_rows, 50);
|
|
model.Add<SigmoidLayer<> >();
|
|
model.Add<Linear<> >(50, 10);
|
|
model.Add<LogSoftMax<> >();
|
|
|
|
ens::VanillaUpdate opt;
|
|
model.ResetParameters();
|
|
#if ENS_VERSION_MAJOR == 1
|
|
opt.Initialize(model.Parameters().n_rows, model.Parameters().n_cols);
|
|
#else
|
|
ens::VanillaUpdate::Policy<arma::mat, arma::mat> optPolicy(opt,
|
|
model.Parameters().n_rows, model.Parameters().n_cols);
|
|
#endif
|
|
double stepSize = 0.01;
|
|
size_t batchSize = 10;
|
|
|
|
size_t iteration = 0;
|
|
bool converged = false;
|
|
while (iteration < 100)
|
|
{
|
|
arma::running_stat<double> error;
|
|
size_t batchStart = 0;
|
|
while (batchStart < dataset.n_cols)
|
|
{
|
|
size_t batchEnd = std::min(batchStart + batchSize,
|
|
(size_t) dataset.n_cols);
|
|
arma::mat currentData = dataset.cols(batchStart, batchEnd - 1);
|
|
arma::mat currentLabels = labels.cols(batchStart, batchEnd - 1);
|
|
arma::mat currentResuls;
|
|
model.Forward(currentData, currentResuls);
|
|
arma::mat gradients;
|
|
model.Backward(currentData, currentLabels, gradients);
|
|
#if ENS_VERSION_MAJOR == 1
|
|
opt.Update(model.Parameters(), stepSize, gradients);
|
|
#else
|
|
optPolicy.Update(model.Parameters(), stepSize, gradients);
|
|
#endif
|
|
batchStart = batchEnd;
|
|
|
|
arma::mat prediction = arma::zeros<arma::mat>(1, currentResuls.n_cols);
|
|
|
|
for (size_t i = 0; i < currentResuls.n_cols; ++i)
|
|
{
|
|
prediction(i) = arma::as_scalar(arma::find(
|
|
arma::max(currentResuls.col(i)) == currentResuls.col(i), 1)) + 1;
|
|
}
|
|
|
|
size_t correct = arma::accu(prediction == currentLabels);
|
|
error(1 - (double) correct / batchSize);
|
|
}
|
|
Log::Debug << "Current training error: " << error.mean() << std::endl;
|
|
iteration++;
|
|
if (error.mean() < 0.05)
|
|
{
|
|
converged = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
REQUIRE(converged);
|
|
}
|
|
|
|
/**
|
|
* Train the dropout network on a larger dataset.
|
|
*/
|
|
TEST_CASE("DropoutNetworkTest", "[FeedForwardNetworkTest]")
|
|
{
|
|
// Load the dataset.
|
|
arma::mat trainData;
|
|
data::Load("thyroid_train.csv", trainData, true);
|
|
|
|
arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
|
|
trainData.shed_row(trainData.n_rows - 1);
|
|
|
|
arma::mat testData;
|
|
data::Load("thyroid_test.csv", testData, true);
|
|
|
|
arma::mat testLabels = testData.row(testData.n_rows - 1);
|
|
testData.shed_row(testData.n_rows - 1);
|
|
|
|
/*
|
|
* Construct a feed forward network with trainData.n_rows input nodes,
|
|
* hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The
|
|
* network structure looks like:
|
|
*
|
|
* Input Hidden Dropout Output
|
|
* Layer Layer Layer Layer
|
|
* +-----+ +-----+ +-----+ +-----+
|
|
* | | | | | | | |
|
|
* | +------>| +------>| +------>| |
|
|
* | | +>| | | | | |
|
|
* +-----+ | +--+--+ +-----+ +-----+
|
|
* |
|
|
* Bias |
|
|
* Layer |
|
|
* +-----+ |
|
|
* | | |
|
|
* | +-----+
|
|
* | |
|
|
* +-----+
|
|
*/
|
|
|
|
FFN<NegativeLogLikelihood<> > model;
|
|
model.Add<Linear<> >(trainData.n_rows, 8);
|
|
model.Add<SigmoidLayer<> >();
|
|
model.Add<Dropout<> >();
|
|
model.Add<Linear<> >(8, 3);
|
|
model.Add<LogSoftMax<> >();
|
|
|
|
// Vanilla neural net with logistic activation function.
|
|
// Because 92% of the patients are not hyperthyroid the neural
|
|
// network must be significant better than 92%.
|
|
TestNetwork<>(model, trainData, trainLabels, testData, testLabels, 10, 0.1);
|
|
arma::mat dataset;
|
|
dataset.load("mnist_first250_training_4s_and_9s.arm");
|
|
|
|
// Normalize each point since these are images.
|
|
for (size_t i = 0; i < dataset.n_cols; ++i)
|
|
{
|
|
dataset.col(i) /= norm(dataset.col(i), 2);
|
|
}
|
|
|
|
arma::mat labels = arma::zeros(1, dataset.n_cols);
|
|
labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1);
|
|
labels += 1;
|
|
|
|
FFN<NegativeLogLikelihood<> > model1;
|
|
model1.Add<Linear<> >(dataset.n_rows, 10);
|
|
model1.Add<SigmoidLayer<> >();
|
|
model.Add<Dropout<> >();
|
|
model1.Add<Linear<> >(10, 2);
|
|
model1.Add<LogSoftMax<> >();
|
|
// Vanilla neural net with logistic activation function.
|
|
TestNetwork<>(model1, dataset, labels, dataset, labels, 10, 0.2);
|
|
}
|
|
|
|
/**
|
|
* Train the highway network on a larger dataset.
|
|
*/
|
|
TEST_CASE("HighwayNetworkTest", "[FeedForwardNetworkTest]")
|
|
{
|
|
arma::mat dataset;
|
|
dataset.load("mnist_first250_training_4s_and_9s.arm");
|
|
|
|
// Normalize each point since these are images.
|
|
for (size_t i = 0; i < dataset.n_cols; ++i)
|
|
dataset.col(i) /= norm(dataset.col(i), 2);
|
|
|
|
arma::mat labels = arma::zeros(1, dataset.n_cols);
|
|
labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1);
|
|
labels += 1;
|
|
|
|
FFN<NegativeLogLikelihood<> > model;
|
|
model.Add<Linear<> >(dataset.n_rows, 10);
|
|
Highway<>* highway = new Highway<>(10, true);
|
|
highway->Add<Linear<> >(10, 10);
|
|
highway->Add<SigmoidLayer<> >();
|
|
model.Add(highway); // This takes ownership of the memory.
|
|
model.Add<Linear<> >(10, 2);
|
|
model.Add<LogSoftMax<> >();
|
|
TestNetwork<>(model, dataset, labels, dataset, labels, 10, 0.2);
|
|
}
|
|
|
|
/**
|
|
* Train the DropConnect network on a larger dataset.
|
|
*/
|
|
TEST_CASE("DropConnectNetworkTest", "[FeedForwardNetworkTest]")
|
|
{
|
|
// Load the dataset.
|
|
arma::mat trainData;
|
|
data::Load("thyroid_train.csv", trainData, true);
|
|
|
|
arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
|
|
trainData.shed_row(trainData.n_rows - 1);
|
|
|
|
arma::mat testData;
|
|
data::Load("thyroid_test.csv", testData, true);
|
|
|
|
arma::mat testLabels = testData.row(testData.n_rows - 1);
|
|
testData.shed_row(testData.n_rows - 1);
|
|
|
|
/*
|
|
* Construct a feed forward network with trainData.n_rows input nodes,
|
|
* hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The
|
|
* network struct that looks like:
|
|
*
|
|
* Input Hidden DropConnect Output
|
|
* Layer Layer Layer Layer
|
|
* +-----+ +-----+ +-----+ +-----+
|
|
* | | | | | | | |
|
|
* | +------>| +------>| +------>| |
|
|
* | | +>| | | | | |
|
|
* +-----+ | +--+--+ +-----+ +-----+
|
|
* |
|
|
* Bias |
|
|
* Layer |
|
|
* +-----+ |
|
|
* | | |
|
|
* | +-----+
|
|
* | |
|
|
* +-----+
|
|
*
|
|
*
|
|
*/
|
|
|
|
FFN<NegativeLogLikelihood<> > model;
|
|
model.Add<Linear<> >(trainData.n_rows, 8);
|
|
model.Add<SigmoidLayer<> >();
|
|
model.Add<DropConnect<> >(8, 3);
|
|
model.Add<LogSoftMax<> >();
|
|
|
|
// Vanilla neural net with logistic activation function.
|
|
// Because 92% of the patients are not hyperthyroid the neural
|
|
// network must be significant better than 92%.
|
|
TestNetwork<>(model, trainData, trainLabels, testData, testLabels, 10, 0.1);
|
|
|
|
arma::mat dataset;
|
|
dataset.load("mnist_first250_training_4s_and_9s.arm");
|
|
|
|
// Normalize each point since these are images.
|
|
for (size_t i = 0; i < dataset.n_cols; ++i)
|
|
dataset.col(i) /= norm(dataset.col(i), 2);
|
|
|
|
arma::mat labels = arma::zeros(1, dataset.n_cols);
|
|
labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1);
|
|
labels += 1;
|
|
|
|
FFN<NegativeLogLikelihood<> > model1;
|
|
model1.Add<Linear<> >(dataset.n_rows, 10);
|
|
model1.Add<SigmoidLayer<> >();
|
|
model1.Add<DropConnect<> >(10, 2);
|
|
model1.Add<LogSoftMax<> >();
|
|
// Vanilla neural net with logistic activation function.
|
|
TestNetwork<>(model1, dataset, labels, dataset, labels, 10, 0.2);
|
|
}
|
|
|
|
/**
|
|
* Test miscellaneous things of FFN,
|
|
* e.g. copy/move constructor, assignment operator.
|
|
*/
|
|
TEST_CASE("FFNMiscTest", "[FeedForwardNetworkTest]")
|
|
{
|
|
FFN<MeanSquaredError<>> model;
|
|
model.Add<Linear<>>(2, 3);
|
|
model.Add<ReLULayer<>>();
|
|
|
|
auto copiedModel(model);
|
|
copiedModel = model;
|
|
auto movedModel(std::move(model));
|
|
auto moveOperator = std::move(copiedModel);
|
|
}
|
|
|
|
/**
|
|
* Test that serialization works ok.
|
|
*/
|
|
TEST_CASE("FFSerializationTest", "[FeedForwardNetworkTest]")
|
|
{
|
|
// Load the dataset.
|
|
arma::mat trainData;
|
|
data::Load("thyroid_train.csv", trainData, true);
|
|
|
|
arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
|
|
trainData.shed_row(trainData.n_rows - 1);
|
|
|
|
arma::mat testData;
|
|
data::Load("thyroid_test.csv", testData, true);
|
|
|
|
arma::mat testLabels = testData.row(testData.n_rows - 1);
|
|
testData.shed_row(testData.n_rows - 1);
|
|
|
|
// Vanilla neural net with logistic activation function.
|
|
// Because 92% of the patients are not hyperthyroid the neural
|
|
// network must be significant better than 92%.
|
|
FFN<NegativeLogLikelihood<> > model;
|
|
model.Add<Linear<> >(trainData.n_rows, 8);
|
|
model.Add<SigmoidLayer<> >();
|
|
model.Add<Dropout<> >();
|
|
model.Add<Linear<> >(8, 3);
|
|
model.Add<LogSoftMax<> >();
|
|
|
|
ens::RMSProp opt(0.01, 32, 0.88, 1e-8, trainData.n_cols /* 1 epoch */, -1);
|
|
|
|
model.Train(trainData, trainLabels, opt);
|
|
|
|
FFN<NegativeLogLikelihood<>> xmlModel, jsonModel, binaryModel;
|
|
xmlModel.Add<Linear<>>(10, 10); // Layer that will get removed.
|
|
|
|
// Serialize into other models.
|
|
SerializeObjectAll(model, xmlModel, jsonModel, binaryModel);
|
|
|
|
arma::mat predictions, xmlPredictions, jsonPredictions, binaryPredictions;
|
|
model.Predict(testData, predictions);
|
|
xmlModel.Predict(testData, xmlPredictions);
|
|
jsonModel.Predict(testData, jsonPredictions);
|
|
jsonModel.Predict(testData, binaryPredictions);
|
|
|
|
CheckMatrices(predictions, xmlPredictions, jsonPredictions,
|
|
binaryPredictions);
|
|
}
|
|
|
|
/**
|
|
* Test if the custom layers work. The target is to see if the code compiles
|
|
* when the Train and Prediction are called.
|
|
*/
|
|
TEST_CASE("CustomLayerTest", "[FeedForwardNetworkTest]")
|
|
{
|
|
// Load the dataset.
|
|
arma::mat trainData;
|
|
data::Load("thyroid_train.csv", trainData, true);
|
|
|
|
arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
|
|
trainData.shed_row(trainData.n_rows - 1);
|
|
|
|
arma::mat testData;
|
|
data::Load("thyroid_test.csv", testData, true);
|
|
|
|
arma::mat testLabels = testData.row(testData.n_rows - 1);
|
|
testData.shed_row(testData.n_rows - 1);
|
|
|
|
FFN<NegativeLogLikelihood<>, RandomInitialization, CustomLayer<> > model;
|
|
model.Add<Linear<> >(trainData.n_rows, 8);
|
|
model.Add<CustomLayer<> >();
|
|
model.Add<Linear<> >(8, 3);
|
|
model.Add<LogSoftMax<> >();
|
|
|
|
ens::RMSProp opt(0.01, 32, 0.88, 1e-8, 15, -1);
|
|
model.Train(trainData, trainLabels, opt);
|
|
|
|
arma::mat predictionTemp;
|
|
model.Predict(testData, predictionTemp);
|
|
arma::mat prediction = arma::zeros<arma::mat>(1, predictionTemp.n_cols);
|
|
}
|
|
|
|
/**
|
|
* Test the overload of Forward function which allows partial forward pass.
|
|
*/
|
|
TEST_CASE("PartialForwardTest", "[FeedForwardNetworkTest]")
|
|
{
|
|
FFN<NegativeLogLikelihood<>, RandomInitialization> model;
|
|
model.Add<Linear<> >(5, 10);
|
|
|
|
// Add a new Add<> module which adds a constant term to the input.
|
|
Add<>* addModule = new Add<>(10);
|
|
model.Add(addModule);
|
|
|
|
LinearNoBias<>* linearNoBiasModule = new LinearNoBias<>(10, 10);
|
|
model.Add(linearNoBiasModule);
|
|
|
|
model.Add<Linear<> >(10, 10);
|
|
|
|
model.ResetParameters();
|
|
// Set the parameters of the Add<> module to a matrix of ones.
|
|
addModule->Parameters() = arma::ones(10, 1);
|
|
// Set the parameters of the LinearNoBias<> module to a matrix of ones.
|
|
linearNoBiasModule->Parameters() = arma::ones(10, 10);
|
|
|
|
arma::mat input = arma::ones(10, 1);
|
|
arma::mat output;
|
|
|
|
// Forward pass only through the Add module.
|
|
model.Forward(input,
|
|
output,
|
|
1 /* Index of the Add module */,
|
|
1 /* Index of the Add module */);
|
|
|
|
// As we only forward pass through Add module, input and output should
|
|
// differ by a matrix of ones.
|
|
CheckMatrices(input, output - 1);
|
|
|
|
// Forward pass only through the Add module and the LinearNoBias module.
|
|
model.Forward(input,
|
|
output,
|
|
1 /* Index of the Add module */,
|
|
2 /* Index of the LinearNoBias module */);
|
|
|
|
// As we only forward pass through Add module followed by the LinearNoBias
|
|
// module, output should be a matrix of 20s.(output = weight * input)
|
|
CheckMatrices(output, arma::ones(10, 1) * 20);
|
|
}
|
|
|
|
/**
|
|
* Test that FFN::Train() returns finite objective value.
|
|
*/
|
|
TEST_CASE("FFNTrainReturnObjective", "[FeedForwardNetworkTest]")
|
|
{
|
|
// Load the dataset.
|
|
arma::mat trainData;
|
|
data::Load("thyroid_train.csv", trainData, true);
|
|
|
|
arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
|
|
trainData.shed_row(trainData.n_rows - 1);
|
|
|
|
arma::mat testData;
|
|
data::Load("thyroid_test.csv", testData, true);
|
|
|
|
arma::mat testLabels = testData.row(testData.n_rows - 1);
|
|
testData.shed_row(testData.n_rows - 1);
|
|
|
|
// Vanilla neural net with logistic activation function.
|
|
// Because 92% of the patients are not hyperthyroid the neural
|
|
// network must be significantly better than 92%.
|
|
FFN<NegativeLogLikelihood<> > model;
|
|
model.Add<Linear<> >(trainData.n_rows, 8);
|
|
model.Add<SigmoidLayer<> >();
|
|
model.Add<Dropout<> >();
|
|
model.Add<Linear<> >(8, 3);
|
|
model.Add<LogSoftMax<> >();
|
|
|
|
ens::RMSProp opt(0.01, 32, 0.88, 1e-8, trainData.n_cols /* 1 epoch */, -1);
|
|
|
|
double objVal = model.Train(trainData, trainLabels, opt);
|
|
|
|
REQUIRE(std::isfinite(objVal) == true);
|
|
}
|
|
|
|
/**
|
|
* Test that FFN::Model() allows us to access the instantiated network.
|
|
*/
|
|
TEST_CASE("FFNReturnModel", "[FeedForwardNetworkTest]")
|
|
{
|
|
// Create dummy network.
|
|
FFN<NegativeLogLikelihood<> > model;
|
|
Linear<>* linearA = new Linear<>(3, 3);
|
|
model.Add(linearA);
|
|
Linear<>* linearB = new Linear<>(3, 4);
|
|
model.Add(linearB);
|
|
|
|
// Initialize network parameter.
|
|
model.ResetParameters();
|
|
|
|
// Set all network parameter to one.
|
|
model.Parameters().ones();
|
|
|
|
// Zero the second layer parameter.
|
|
linearB->Parameters().zeros();
|
|
|
|
// Get the layer parameter from layer A and layer B and store them in
|
|
// parameterA and parameterB.
|
|
arma::mat parameterA, parameterB;
|
|
boost::apply_visitor(ParametersVisitor(parameterA), model.Model()[0]);
|
|
boost::apply_visitor(ParametersVisitor(parameterB), model.Model()[1]);
|
|
|
|
CheckMatrices(parameterA, arma::ones(3 * 3 + 3, 1));
|
|
CheckMatrices(parameterB, arma::zeros(3 * 4 + 4, 1));
|
|
|
|
CheckMatrices(linearA->Parameters(), arma::ones(3 * 3 + 3, 1));
|
|
CheckMatrices(linearB->Parameters(), arma::zeros(3 * 4 + 4, 1));
|
|
}
|
|
|
|
/**
|
|
* Test to see if the FFN code compiles when the Optimizer
|
|
* doesn't have the MaxIterations() method.
|
|
*/
|
|
TEST_CASE("OptimizerTest", "[FeedForwardNetworkTest]")
|
|
{
|
|
// Load the dataset.
|
|
arma::mat trainData;
|
|
data::Load("thyroid_train.csv", trainData, true);
|
|
|
|
arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
|
|
trainData.shed_row(trainData.n_rows - 1);
|
|
|
|
arma::mat testData;
|
|
data::Load("thyroid_test.csv", testData, true);
|
|
|
|
arma::mat testLabels = testData.row(testData.n_rows - 1);
|
|
testData.shed_row(testData.n_rows - 1);
|
|
|
|
FFN<NegativeLogLikelihood<>, RandomInitialization, CustomLayer<> > model;
|
|
model.Add<Linear<> >(trainData.n_rows, 8);
|
|
model.Add<CustomLayer<> >();
|
|
model.Add<Linear<> >(8, 3);
|
|
model.Add<LogSoftMax<> >();
|
|
|
|
ens::DE opt(200, 1000, 0.6, 0.8, 1e-5);
|
|
model.Train(trainData, trainLabels, opt);
|
|
}
|
|
|
|
/**
|
|
* Test to see if an exception is thrown when input with
|
|
* wrong shape is provided to a FFN.
|
|
*/
|
|
TEST_CASE("FFNCheckInputShapeTest", "[FeedForwardNetworkTest]")
|
|
{
|
|
// Load the dataset.
|
|
arma::mat trainData;
|
|
data::Load("thyroid_train.csv", trainData, true);
|
|
|
|
arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
|
|
trainData.shed_row(trainData.n_rows - 1);
|
|
|
|
arma::mat testData;
|
|
data::Load("thyroid_test.csv", testData, true);
|
|
|
|
arma::mat testLabels = testData.row(testData.n_rows - 1);
|
|
testData.shed_row(testData.n_rows - 1);
|
|
|
|
FFN<NegativeLogLikelihood<>, RandomInitialization, CustomLayer<> > model;
|
|
// Purposely putting wrong input shape so that error is thrown.
|
|
model.Add<Linear<> >(trainData.n_rows - 3, 8);
|
|
model.Add<CustomLayer<> >();
|
|
model.Add<Linear<> >(8, 3);
|
|
model.Add<LogSoftMax<> >();
|
|
|
|
std::string expectedMsg = "FFN<>::Train(): ";
|
|
expectedMsg += "the first layer of the network expects ";
|
|
expectedMsg += std::to_string(trainData.n_rows - 3) + " elements, ";
|
|
expectedMsg += "but the input has " + std::to_string(trainData.n_rows) + " dimensions! ";
|
|
|
|
ens::DE opt(200, 1000, 0.6, 0.8, 1e-5);
|
|
|
|
REQUIRE_THROWS_AS(model.Train(trainData, trainLabels, opt), std::logic_error);
|
|
}
|