apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into cereal

Browse Source 
Signed-off-by: Omar Shrit <omar@shrit.me>
This commit is contained in:
Omar Shrit
2020-10-03 23:09:49 +02:00
parent d6406f4546 c104a2dcf0
commit 8f9fdfd8c6
2 changed files with 97 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mlpack/tests/ann_layer_test.cpp
+93 -108
View File
@@ -87,11 +87,10 @@ TEST_CASE("GradientAddLayerTest", "[ANNLayerTest]")
// Add function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randu(10, 1)),
target(arma::mat("1"))
{
input = arma::randu(10, 1);
target = arma::mat("1");
model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
@@ -416,11 +415,10 @@ TEST_CASE("GradientLinearLayerTest", "[ANNLayerTest]")
// Linear function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randu(10, 1)),
target(arma::mat("1"))
{
input = arma::randu(10, 1);
target = arma::mat("1");
model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
@@ -508,13 +506,12 @@ TEST_CASE("GradientLinear3DLayerTest", "[ANNLayerTest]")
// Linear function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
inSize(4),
outSize(1),
nPoints(2),
batchSize(4)
{
const size_t inSize = 4;
const size_t outSize = 1;
const size_t nPoints = 2;
const size_t batchSize = 4;
input = arma::randu(inSize * nPoints, batchSize);
target = arma::zeros(outSize * nPoints, batchSize);
target(0, 0) = 1;
@@ -545,6 +542,10 @@ TEST_CASE("GradientLinear3DLayerTest", "[ANNLayerTest]")
FFN<MeanSquaredError<>, RandomInitialization>* model;
arma::mat input, target;
const size_t inSize;
const size_t outSize;
const size_t nPoints;
const size_t batchSize;
} function;
REQUIRE(CheckGradient(function) <= 1e-7);
@@ -591,11 +592,10 @@ TEST_CASE("GradientNoisyLinearLayerTest", "[ANNLayerTest]")
// Noisy linear function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randu(10, 1)),
target(arma::mat("1"))
{
input = arma::randu(10, 1);
target = arma::mat("1");
model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
@@ -695,11 +695,10 @@ TEST_CASE("GradientLinearNoBiasLayerTest", "[ANNLayerTest]")
// LinearNoBias function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randu(10, 1)),
target(arma::mat("1"))
{
input = arma::randu(10, 1);
target = arma::mat("1");
model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
@@ -797,11 +796,10 @@ TEST_CASE("GradientFlexibleReLULayerTest", "[ANNLayerTest]")
// Add function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randu(2, 1)),
target(arma::mat("1"))
{
input = arma::randu(2, 1);
target = arma::mat("1");
model = new FFN<NegativeLogLikelihood<>, RandomInitialization>(
NegativeLogLikelihood<>(), RandomInitialization(0.1, 0.5));
@@ -1017,10 +1015,10 @@ TEST_CASE("GradientLSTMLayerTest", "[ANNLayerTest]")
// LSTM function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randu(1, 1, 5)),
target(arma::ones(1, 1, 5))
{
input = arma::randu(1, 1, 5);
target.ones(1, 1, 5);
const size_t rho = 5;
model = new RNN<NegativeLogLikelihood<> >(rho);
@@ -1122,10 +1120,10 @@ TEST_CASE("GradientFastLSTMLayerTest", "[ANNLayerTest]")
// Fast LSTM function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randu(1, 1, 5)),
target(arma::ones(1, 1, 5))
{
input = arma::randu(1, 1, 5);
target = arma::ones(1, 1, 5);
const size_t rho = 5;
model = new RNN<NegativeLogLikelihood<> >(rho);
@@ -1391,10 +1389,10 @@ TEST_CASE("GradientGRULayerTest", "[ANNLayerTest]")
// GRU function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randu(1, 1, 5)),
target(arma::ones(1, 1, 5))
{
input = arma::randu(1, 1, 5);
target = arma::ones(1, 1, 5);
const size_t rho = 5;
model = new RNN<NegativeLogLikelihood<> >(rho);
@@ -1631,11 +1629,10 @@ TEST_CASE("GradientConcatLayerTest", "[ANNLayerTest]")
// Concat function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randu(10, 1)),
target(arma::mat("1"))
{
input = arma::randu(10, 1);
target = arma::mat("1");
model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
@@ -1700,11 +1697,10 @@ TEST_CASE("GradientConcatenateLayerTest", "[ANNLayerTest]")
// Concatenate function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randu(10, 1)),
target(arma::mat("1"))
{
input = arma::randu(10, 1);
target = arma::mat("1");
model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
@@ -1905,11 +1901,10 @@ TEST_CASE("GradientSoftmaxTest", "[ANNLayerTest]")
// Softmax function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randu(10, 1)),
target(arma::mat("1; 0"))
{
input = arma::randu(10, 1);
target = arma::mat("1; 0");
model = new FFN<MeanSquaredError<>, RandomInitialization>;
model->Predictors() = input;
model->Responses() = target;
@@ -2109,12 +2104,10 @@ TEST_CASE("GradientBatchNormTest", "[ANNLayerTest]")
// Add function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randn(32, 2048)),
target(arma::ones(1, 2048))
{
input = arma::randn(32, 2048);
arma::mat target;
target.ones(1, 2048);
model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
@@ -2184,12 +2177,11 @@ TEST_CASE("GradientVirtualBatchNormTest", "[ANNLayerTest]")
// Add function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randn(5, 256)),
target(arma::ones(1, 256))
{
input = arma::randn(5, 256);
arma::mat referenceBatch = arma::mat(input.memptr(), input.n_rows, 16);
arma::mat target;
target.ones(1, 256);
model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
model->Predictors() = input;
@@ -2247,12 +2239,10 @@ TEST_CASE("MiniBatchDiscriminationTest", "[ANNLayerTest]")
// Add function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randn(5, 4)),
target(arma::ones(1, 4))
{
input = arma::randn(5, 4);
arma::mat target;
target.ones(1, 4);
model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
@@ -2427,11 +2417,10 @@ TEST_CASE("GradientTransposedConvolutionLayerTest", "[ANNLayerTest]")
{
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::linspace<arma::colvec>(0, 35, 36)),
target(arma::mat("1"))
{
input = arma::linspace<arma::colvec>(0, 35, 36);
target = arma::mat("1");
model = new FFN<NegativeLogLikelihood<>, RandomInitialization>();
model->Predictors() = input;
model->Responses() = target;
@@ -2544,11 +2533,10 @@ TEST_CASE("GradientAtrousConvolutionLayerTest", "[ANNLayerTest]")
// Add function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::linspace<arma::colvec>(0, 35, 36)),
target(arma::mat("1"))
{
input = arma::linspace<arma::colvec>(0, 35, 36);
target = arma::mat("1");
model = new FFN<NegativeLogLikelihood<>, RandomInitialization>();
model->Predictors() = input;
model->Responses() = target;
@@ -2575,7 +2563,7 @@ TEST_CASE("GradientAtrousConvolutionLayerTest", "[ANNLayerTest]")
arma::mat input, target;
} function;
// TODO: this tolerance seems far higher than necessary. The implementation
// TODO: this tolerance seems far higher than necessary. The implementation
// should be checked.
REQUIRE(CheckGradient(function) <= 0.2);
}
@@ -2726,12 +2714,10 @@ TEST_CASE("GradientLayerNormTest", "[ANNLayerTest]")
// Add function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randn(10, 256)),
target(arma::ones(1, 256))
{
input = arma::randn(10, 256);
arma::mat target;
target.ones(1, 256);
model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
@@ -3048,11 +3034,10 @@ TEST_CASE("GradientReparametrizationLayerTest", "[ANNLayerTest]")
// Linear function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randu(10, 1)),
target(arma::mat("1"))
{
input = arma::randu(10, 1);
target = arma::mat("1");
model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
@@ -3092,11 +3077,10 @@ TEST_CASE("GradientReparametrizationLayerBetaTest", "[ANNLayerTest]")
// Linear function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randu(10, 2)),
target(arma::mat("1 1"))
{
input = arma::randu(10, 2);
target = arma::mat("1 1");
model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
@@ -3248,11 +3232,10 @@ TEST_CASE("GradientHighwayLayerTest", "[ANNLayerTest]")
// Linear function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randu(5, 1)),
target(arma::mat("1"))
{
input = arma::randu(5, 1);
target = arma::mat("1");
model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
@@ -3300,11 +3283,10 @@ TEST_CASE("GradientSequentialLayerTest", "[ANNLayerTest]")
// Linear function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randu(10, 1)),
target(arma::mat("1"))
{
input = arma::randu(10, 1);
target = arma::mat("1");
model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
@@ -3351,11 +3333,10 @@ TEST_CASE("GradientWeightNormLayerTest", "[ANNLayerTest]")
// Linear function gradient instantiation.
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randu(10, 1)),
target(arma::mat("1"))
{
input = arma::randu(10, 1);
target = arma::mat("1");
model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
@@ -4185,12 +4166,10 @@ TEST_CASE("GradientBatchNormWithMiniBatchesTest", "[ANNLayerTest]")
{
struct GradientFunction
{
GradientFunction()
GradientFunction() :
input(arma::randn(16, 1024)),
target(arma::ones(1, 1024))
{
input = arma::randn(16, 1024);
arma::mat target;
target.ones(1, 1024);
model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
@@ -4683,7 +4662,13 @@ TEST_CASE("GradientMultiheadAttentionTest", "[ANNLayerTest]")
{
struct GradientFunction
{
GradientFunction()
GradientFunction() :
tgtSeqLen(2),
srcSeqLen(2),
embedDim(4),
nHeads(2),
vocabSize(5),
batchSize(2)
{
input = arma::randu(embedDim * (tgtSeqLen + 2 * srcSeqLen), batchSize);
target = arma::zeros(vocabSize, batchSize);
@@ -4736,13 +4721,13 @@ TEST_CASE("GradientMultiheadAttentionTest", "[ANNLayerTest]")
MultiheadAttention<>* attnModule;
arma::mat input, target, attnMask, keyPaddingMask;
const size_t tgtSeqLen = 2;
const size_t srcSeqLen = 2;
const size_t embedDim = 4;
const size_t nHeads = 2;
const size_t vocabSize = 5;
const size_t batchSize = 2;
const size_t tgtSeqLen;
const size_t srcSeqLen;
const size_t embedDim;
const size_t nHeads;
const size_t vocabSize;
const size_t batchSize;
} function;
REQUIRE(CheckGradient(function) <= 2e-06);
REQUIRE(CheckGradient(function) <= 3e-06);
}
src/mlpack/tests/feedforward_network_test.cpp
+4 -3
View File
@@ -146,10 +146,10 @@ TEST_CASE("CheckCopyMovingVanillaNetworkTest", "[FeedForwardNetworkTest]")
model1->Add<Linear<> >(8, 3);
model1->Add<LogSoftMax<> >();
// Check whether copy cpnstructor is working or not.
// Check whether copy constructor is working or not.
CheckCopyFunction<>(model, trainData, trainLabels, 1);
// Check whether move cpnstructor is working or not.
// Check whether move constructor is working or not.
CheckMoveFunction<>(model1, trainData, trainLabels, 1);
}
@@ -487,7 +487,7 @@ TEST_CASE("FFNMiscTest", "[FeedForwardNetworkTest]")
auto copiedModel(model);
copiedModel = model;
auto movedModel(std::move(model));
movedModel = std::move(copiedModel);
auto moveOperator = std::move(copiedModel);
}
/**
@@ -715,3 +715,4 @@ TEST_CASE("OptimizerTest", "[FeedForwardNetworkTest]")
ens::DE opt(200, 1000, 0.6, 0.8, 1e-5);
model.Train(trainData, trainLabels, opt);
}