Update FFN tests to use the base layer class.

src/mlpack/methods/ann/layer/add.hpp

@@ -13,6 +13,7 @@
 #define MLPACK_METHODS_ANN_LAYER_ADD_HPP
 
 #include <mlpack/prereqs.hpp>
+#include "layer.hpp"
 #include <mlpack/methods/ann/layer/layer_traits.hpp>
 
 namespace mlpack {
@@ -41,6 +42,9 @@ class AddType : public Layer<InputType, OutputType>
    */
   AddType(const size_t outSize = 0);
 
+  //! Clone the AddType object. This handles polymorphism correctly.
+  AddType* Clone() const { return new AddType(*this); }
+
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function
    * f(x) by propagating the activity forward through f.

src/mlpack/methods/ann/layer/base_layer.hpp

@@ -171,6 +171,7 @@ typedef BaseLayer<IdentityFunction, arma::mat, arma::mat> Identity;
 // using ReLULayer = BaseLayer<
 //     ActivationFunction, InputDataType, OutputDataType>;
 typedef BaseLayer<RectifierFunction, arma::mat, arma::mat> ReLULayer;
+typedef BaseLayer<RectifierFunction, arma::mat, arma::mat> ReLU;
 
 /**
  * Standard hyperbolic tangent layer.

src/mlpack/methods/ann/layer/celu.hpp

@@ -70,7 +70,7 @@ class CELUType : public Layer<InputType, OutputType>
   CELUType(const double alpha = 1.0);
 
   //! Clone the CELUType object. This handles polymorphism correctly.
-	CELUType* Clone() const { return new CELUType(*this); }
+  CELUType* Clone() const { return new CELUType(*this); }
 
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function

src/mlpack/methods/ann/layer/dropconnect_impl.hpp

@@ -17,6 +17,8 @@
 // In case it hasn't yet been included.
 #include "dropconnect.hpp"
 
+#include "linear.hpp"
+
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 

src/mlpack/methods/ann/layer/highway.hpp

@@ -65,8 +65,8 @@ class HighwayType : public Layer<InputType, OutputType>
   //! Destroy the Highway object.
   ~HighwayType();
 
-	//! Clone the HighwayType object. This handles polymorphism correctly.
-	HighwayType* Clone() const { return new HighwayType(*this); }
+  //! Clone the HighwayType object. This handles polymorphism correctly.
+  HighwayType* Clone() const { return new HighwayType(*this); }
 
   /**
    * Reset the layer parameter.

src/mlpack/methods/ann/layer/layer_types.hpp

@@ -26,6 +26,7 @@
 #include <mlpack/methods/ann/layer/concat.hpp>
 #include <mlpack/methods/ann/layer/concatenate.hpp>
 #include <mlpack/methods/ann/layer/convolution.hpp>
+#include <mlpack/methods/ann/layer/dropconnect.hpp>
 #include <mlpack/methods/ann/layer/dropout.hpp>
 #include <mlpack/methods/ann/layer/elu.hpp>
 #include <mlpack/methods/ann/layer/glimpse.hpp>

src/mlpack/tests/CMakeLists.txt

@@ -5,7 +5,7 @@ add_executable(mlpack_test
   # akfn_test.cpp
   # aknn_test.cpp
   # ann_dist_test.cpp
-  ann_layer_test.cpp
+  # ann_layer_test.cpp
   # ann_regularizer_test.cpp
   # ann_test_tools.hpp
   # ann_visitor_test.cpp

src/mlpack/tests/feedforward_network_test.cpp

@@ -337,3 +337,569 @@ TEST_CASE("CheckCopyMovingDropoutNetworkTest", "[FeedForwardNetworkTest]")
   // Check whether move constructor is working or not.
   CheckMoveFunction(model1, trainData, trainLabels, 1);
 }
+
+/**
+ * Train the vanilla network on a larger dataset.
+ */
+TEST_CASE("FFVanillaNetworkTest", "[FeedForwardNetworkTest]")
+{
+  // Load the dataset.
+  arma::mat trainData;
+  data::Load("thyroid_train.csv", trainData, true);
+
+  arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
+  trainData.shed_row(trainData.n_rows - 1);
+
+  arma::mat testData;
+  data::Load("thyroid_test.csv", testData, true);
+
+  arma::mat testLabels = testData.row(testData.n_rows - 1);
+  testData.shed_row(testData.n_rows - 1);
+
+  /*
+   * Construct a feed forward network with trainData.n_rows input nodes,
+   * hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The
+   * network structure looks like:
+   *
+   *  Input         Hidden        Output
+   *  Layer         Layer         Layer
+   * +-----+       +-----+       +-----+
+   * |     |       |     |       |     |
+   * |     +------>|     +------>|     |
+   * |     |     +>|     |     +>|     |
+   * +-----+     | +--+--+     | +-----+
+   *             |             |
+   *  Bias       |  Bias       |
+   *  Layer      |  Layer      |
+   * +-----+     | +-----+     |
+   * |     |     | |     |     |
+   * |     +-----+ |     +-----+
+   * |     |       |     |
+   * +-----+       +-----+
+   */
+
+  FFN<NegativeLogLikelihood<> > model;
+  model.Add<Linear>(trainData.n_rows, 8);
+  model.Add<Sigmoid>();
+  model.Add<Linear>(8, 3);
+  model.Add<LogSoftMax>();
+
+  // Vanilla neural net with logistic activation function.
+  // Because 92% of the patients are not hyperthyroid the neural
+  // network must be significant better than 92%.
+  TestNetwork<>(model, trainData, trainLabels, testData, testLabels, 10, 0.1);
+
+  arma::mat dataset;
+  dataset.load("mnist_first250_training_4s_and_9s.arm");
+
+  // Normalize each point since these are images.
+  for (size_t i = 0; i < dataset.n_cols; ++i)
+    dataset.col(i) /= norm(dataset.col(i), 2);
+
+  arma::mat labels = arma::zeros(1, dataset.n_cols);
+  labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1);
+  labels += 1;
+
+  FFN<NegativeLogLikelihood<> > model1;
+  model1.Add<Linear>(dataset.n_rows, 10);
+  model1.Add<Sigmoid>();
+  model1.Add<Linear>(10, 2);
+  model1.Add<LogSoftMax>();
+  // Vanilla neural net with logistic activation function.
+  TestNetwork(model1, dataset, labels, dataset, labels, 10, 0.2);
+}
+
+TEST_CASE("ForwardBackwardTest", "[FeedForwardNetworkTest]")
+{
+  arma::mat dataset;
+  dataset.load("mnist_first250_training_4s_and_9s.arm");
+
+  // Normalize each point since these are images.
+  for (size_t i = 0; i < dataset.n_cols; ++i)
+    dataset.col(i) /= norm(dataset.col(i), 2);
+
+  arma::mat labels = arma::zeros(1, dataset.n_cols);
+  labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1);
+  labels += 1;
+
+  FFN<NegativeLogLikelihood<> > model;
+  model.Add<Linear>(dataset.n_rows, 50);
+  model.Add<Sigmoid>();
+  model.Add<Linear>(50, 10);
+  model.Add<LogSoftMax>();
+
+  ens::VanillaUpdate opt;
+  model.ResetParameters();
+  #if ENS_VERSION_MAJOR == 1
+  opt.Initialize(model.Parameters().n_rows, model.Parameters().n_cols);
+  #else
+  ens::VanillaUpdate::Policy<arma::mat, arma::mat> optPolicy(opt,
+      model.Parameters().n_rows, model.Parameters().n_cols);
+  #endif
+  double stepSize = 0.01;
+  size_t batchSize = 10;
+
+  size_t iteration = 0;
+  bool converged = false;
+  while (iteration < 100)
+  {
+    arma::running_stat<double> error;
+    size_t batchStart = 0;
+    while (batchStart < dataset.n_cols)
+    {
+      size_t batchEnd = std::min(batchStart + batchSize,
+          (size_t) dataset.n_cols);
+      arma::mat currentData = dataset.cols(batchStart, batchEnd - 1);
+      arma::mat currentLabels = labels.cols(batchStart, batchEnd - 1);
+      arma::mat currentResuls;
+      model.Forward(currentData, currentResuls);
+      arma::mat gradients;
+      model.Backward(currentData, currentLabels, gradients);
+      #if ENS_VERSION_MAJOR == 1
+      opt.Update(model.Parameters(), stepSize, gradients);
+      #else
+      optPolicy.Update(model.Parameters(), stepSize, gradients);
+      #endif
+      batchStart = batchEnd;
+
+      arma::mat prediction = arma::zeros<arma::mat>(1, currentResuls.n_cols);
+
+      for (size_t i = 0; i < currentResuls.n_cols; ++i)
+      {
+        prediction(i) = arma::as_scalar(arma::find(
+            arma::max(currentResuls.col(i)) == currentResuls.col(i), 1)) + 1;
+      }
+
+      size_t correct = arma::accu(prediction == currentLabels);
+      error(1 - (double) correct / batchSize);
+    }
+    Log::Debug << "Current training error: " << error.mean() << std::endl;
+    iteration++;
+    if (error.mean() < 0.05)
+    {
+      converged = true;
+      break;
+    }
+  }
+
+  REQUIRE(converged);
+}
+
+/**
+ * Train the dropout network on a larger dataset.
+ */
+TEST_CASE("DropoutNetworkTest", "[FeedForwardNetworkTest]")
+{
+  // Load the dataset.
+  arma::mat trainData;
+  data::Load("thyroid_train.csv", trainData, true);
+
+  arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
+  trainData.shed_row(trainData.n_rows - 1);
+
+  arma::mat testData;
+  data::Load("thyroid_test.csv", testData, true);
+
+  arma::mat testLabels = testData.row(testData.n_rows - 1);
+  testData.shed_row(testData.n_rows - 1);
+
+  /*
+   * Construct a feed forward network with trainData.n_rows input nodes,
+   * hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The
+   * network structure looks like:
+   *
+   *  Input         Hidden        Dropout      Output
+   *  Layer         Layer         Layer        Layer
+   * +-----+       +-----+       +-----+       +-----+
+   * |     |       |     |       |     |       |     |
+   * |     +------>|     +------>|     +------>|     |
+   * |     |     +>|     |       |     |       |     |
+   * +-----+     | +--+--+       +-----+       +-----+
+   *             |
+   *  Bias       |
+   *  Layer      |
+   * +-----+     |
+   * |     |     |
+   * |     +-----+
+   * |     |
+   * +-----+
+   */
+
+  FFN<NegativeLogLikelihood<> > model;
+  model.Add<Linear>(trainData.n_rows, 8);
+  model.Add<Sigmoid>();
+  model.Add<Dropout>();
+  model.Add<Linear>(8, 3);
+  model.Add<LogSoftMax>();
+
+  // Vanilla neural net with logistic activation function.
+  // Because 92% of the patients are not hyperthyroid the neural
+  // network must be significant better than 92%.
+  TestNetwork<>(model, trainData, trainLabels, testData, testLabels, 10, 0.1);
+  arma::mat dataset;
+  dataset.load("mnist_first250_training_4s_and_9s.arm");
+
+  // Normalize each point since these are images.
+  for (size_t i = 0; i < dataset.n_cols; ++i)
+  {
+    dataset.col(i) /= norm(dataset.col(i), 2);
+  }
+
+  arma::mat labels = arma::zeros(1, dataset.n_cols);
+  labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1);
+  labels += 1;
+
+  FFN<NegativeLogLikelihood<> > model1;
+  model1.Add<Linear>(dataset.n_rows, 10);
+  model1.Add<Sigmoid>();
+  model.Add<Dropout>();
+  model1.Add<Linear>(10, 2);
+  model1.Add<LogSoftMax>();
+  // Vanilla neural net with logistic activation function.
+  TestNetwork(model1, dataset, labels, dataset, labels, 10, 0.2);
+}
+
+/**
+ * Train the highway network on a larger dataset.
+ */
+TEST_CASE("HighwayNetworkTest", "[FeedForwardNetworkTest]")
+{
+  arma::mat dataset;
+  dataset.load("mnist_first250_training_4s_and_9s.arm");
+
+  // Normalize each point since these are images.
+  for (size_t i = 0; i < dataset.n_cols; ++i)
+    dataset.col(i) /= norm(dataset.col(i), 2);
+
+  arma::mat labels = arma::zeros(1, dataset.n_cols);
+  labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1);
+  labels += 1;
+
+  FFN<NegativeLogLikelihood<> > model;
+  model.Add<Linear>(dataset.n_rows, 10);
+  Highway* highway = new Highway(10, true);
+  highway->Add<Linear>(10, 10);
+  highway->Add<Sigmoid>();
+  model.Add(highway); // This takes ownership of the memory.
+  model.Add<Linear>(10, 2);
+  model.Add<LogSoftMax>();
+  TestNetwork(model, dataset, labels, dataset, labels, 10, 0.2);
+}
+
+/**
+ * Train the DropConnect network on a larger dataset.
+ */
+TEST_CASE("DropConnectNetworkTest", "[FeedForwardNetworkTest]")
+{
+  // Load the dataset.
+  arma::mat trainData;
+  data::Load("thyroid_train.csv", trainData, true);
+
+  arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
+  trainData.shed_row(trainData.n_rows - 1);
+
+  arma::mat testData;
+  data::Load("thyroid_test.csv", testData, true);
+
+  arma::mat testLabels = testData.row(testData.n_rows - 1);
+  testData.shed_row(testData.n_rows - 1);
+
+ /*
+  *  Construct a feed forward network with trainData.n_rows input nodes,
+  *  hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The
+  *  network struct that looks like:
+  *
+  *  Input         Hidden     DropConnect     Output
+  *  Layer         Layer         Layer        Layer
+  * +-----+       +-----+       +-----+       +-----+
+  * |     |       |     |       |     |       |     |
+  * |     +------>|     +------>|     +------>|     |
+  * |     |     +>|     |       |     |       |     |
+  * +-----+     | +--+--+       +-----+       +-----+
+  *             |
+  *  Bias       |
+  *  Layer      |
+  * +-----+     |
+  * |     |     |
+  * |     +-----+
+  * |     |
+  * +-----+
+  *
+  *
+  */
+
+  FFN<NegativeLogLikelihood<> > model;
+  model.Add<Linear>(trainData.n_rows, 8);
+  model.Add<Sigmoid>();
+  model.Add<DropConnect>(8, 3);
+  model.Add<LogSoftMax>();
+
+  // Vanilla neural net with logistic activation function.
+  // Because 92% of the patients are not hyperthyroid the neural
+  // network must be significant better than 92%.
+  TestNetwork(model, trainData, trainLabels, testData, testLabels, 10, 0.1);
+
+  arma::mat dataset;
+  dataset.load("mnist_first250_training_4s_and_9s.arm");
+
+  // Normalize each point since these are images.
+  for (size_t i = 0; i < dataset.n_cols; ++i)
+    dataset.col(i) /= norm(dataset.col(i), 2);
+
+  arma::mat labels = arma::zeros(1, dataset.n_cols);
+  labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1);
+  labels += 1;
+
+  FFN<NegativeLogLikelihood<> > model1;
+  model1.Add<Linear>(dataset.n_rows, 10);
+  model1.Add<Sigmoid>();
+  model1.Add<DropConnect>(10, 2);
+  model1.Add<LogSoftMax>();
+
+  // Vanilla neural net with logistic activation function.
+  TestNetwork(model1, dataset, labels, dataset, labels, 10, 0.2);
+}
+
+/**
+ * Test miscellaneous things of FFN,
+ * e.g. copy/move constructor, assignment operator.
+ */
+TEST_CASE("FFNMiscTest", "[FeedForwardNetworkTest]")
+{
+  FFN<MeanSquaredError<>> model;
+  model.Add<Linear>(2, 3);
+  model.Add<ReLU>();
+
+  auto copiedModel(model);
+  copiedModel = model;
+  auto movedModel(std::move(model));
+  auto moveOperator = std::move(copiedModel);
+}
+
+/**
+ * Test that serialization works ok.
+ */
+TEST_CASE("FFSerializationTest", "[FeedForwardNetworkTest]")
+{
+  // Load the dataset.
+  arma::mat trainData;
+  data::Load("thyroid_train.csv", trainData, true);
+
+  arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
+  trainData.shed_row(trainData.n_rows - 1);
+
+  arma::mat testData;
+  data::Load("thyroid_test.csv", testData, true);
+
+  arma::mat testLabels = testData.row(testData.n_rows - 1);
+  testData.shed_row(testData.n_rows - 1);
+
+  // Vanilla neural net with logistic activation function.
+  // Because 92% of the patients are not hyperthyroid the neural
+  // network must be significant better than 92%.
+  FFN<NegativeLogLikelihood<> > model;
+  model.Add<Linear>(trainData.n_rows, 8);
+  model.Add<Sigmoid>();
+  model.Add<Dropout>();
+  model.Add<Linear>(8, 3);
+  model.Add<LogSoftMax>();
+
+  ens::RMSProp opt(0.01, 32, 0.88, 1e-8, trainData.n_cols /* 1 epoch */, -1);
+
+  model.Train(trainData, trainLabels, opt);
+
+  FFN<NegativeLogLikelihood<>> xmlModel, jsonModel, binaryModel;
+  xmlModel.Add<Linear>(10, 10); // Layer that will get removed.
+
+  // Serialize into other models.
+  SerializeObjectAll(model, xmlModel, jsonModel, binaryModel);
+
+  arma::mat predictions, xmlPredictions, jsonPredictions, binaryPredictions;
+  model.Predict(testData, predictions);
+  xmlModel.Predict(testData, xmlPredictions);
+  jsonModel.Predict(testData, jsonPredictions);
+  jsonModel.Predict(testData, binaryPredictions);
+
+  // TODO: serialization
+  //CheckMatrices(predictions, xmlPredictions, jsonPredictions,
+  //    binaryPredictions);
+}
+
+/**
+ * Test the overload of Forward function which allows partial forward pass.
+ */
+TEST_CASE("PartialForwardTest", "[FeedForwardNetworkTest]")
+{
+  FFN<NegativeLogLikelihood<>, RandomInitialization> model;
+  model.Add<Linear>(5, 10);
+
+  // Add a new Add<> module which adds a constant term to the input.
+  Add* addModule = new Add(10);
+  model.Add(addModule);
+
+  LinearNoBias* linearNoBiasModule = new LinearNoBias(10, 10);
+  model.Add(linearNoBiasModule);
+
+  model.Add<Linear>(10, 10);
+
+  model.ResetParameters();
+  // Set the parameters of the Add<> module to a matrix of ones.
+  addModule->Parameters() = arma::ones(10, 1);
+  // Set the parameters of the LinearNoBias<> module to a matrix of ones.
+  linearNoBiasModule->Parameters() = arma::ones(10, 10);
+
+  arma::mat input = arma::ones(10, 1);
+  arma::mat output;
+
+  // Forward pass only through the Add module.
+  model.Forward(input,
+                output,
+                1 /* Index of the Add module */,
+                1 /* Index of the Add module */);
+
+  // As we only forward pass through Add module, input and output should
+  // differ by a matrix of ones.
+  CheckMatrices(input, output - 1);
+
+  // Forward pass only through the Add module and the LinearNoBias module.
+  model.Forward(input,
+                output,
+                1 /* Index of the Add module */,
+                2 /* Index of the LinearNoBias module */);
+
+  // As we only forward pass through Add module followed by the LinearNoBias
+  // module, output should be a matrix of 20s.(output = weight * input)
+  CheckMatrices(output, arma::ones(10, 1) * 20);
+}
+
+/**
+ * Test that FFN::Train() returns finite objective value.
+ */
+TEST_CASE("FFNTrainReturnObjective", "[FeedForwardNetworkTest]")
+{
+  // Load the dataset.
+  arma::mat trainData;
+  data::Load("thyroid_train.csv", trainData, true);
+
+  arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
+  trainData.shed_row(trainData.n_rows - 1);
+
+  arma::mat testData;
+  data::Load("thyroid_test.csv", testData, true);
+
+  arma::mat testLabels = testData.row(testData.n_rows - 1);
+  testData.shed_row(testData.n_rows - 1);
+
+  // Vanilla neural net with logistic activation function.
+  // Because 92% of the patients are not hyperthyroid the neural
+  // network must be significantly better than 92%.
+  FFN<NegativeLogLikelihood<> > model;
+  model.Add<Linear>(trainData.n_rows, 8);
+  model.Add<Sigmoid>();
+  model.Add<Dropout>();
+  model.Add<Linear>(8, 3);
+  model.Add<LogSoftMax>();
+
+  ens::RMSProp opt(0.01, 32, 0.88, 1e-8, trainData.n_cols /* 1 epoch */, -1);
+
+  double objVal = model.Train(trainData, trainLabels, opt);
+
+  REQUIRE(std::isfinite(objVal) == true);
+}
+
+/**
+ * Test that FFN::Model() allows us to access the instantiated network.
+ */
+TEST_CASE("FFNReturnModel", "[FeedForwardNetworkTest]")
+{
+  // Create dummy network.
+  FFN<NegativeLogLikelihood<> > model;
+  Linear* linearA = new Linear(3, 3);
+  model.Add(linearA);
+  Linear* linearB = new Linear(3, 4);
+  model.Add(linearB);
+
+  // Initialize network parameter.
+  model.ResetParameters();
+
+  // Set all network parameter to one.
+  model.Parameters().ones();
+
+  // Zero the second layer parameter.
+  linearB->Parameters().zeros();
+
+  // Get the layer parameter from layer A and layer B and store them in
+  // parameterA and parameterB.
+  const arma::mat parameterA = model.Model()[0]->Parameters();
+  const arma::mat parameterB = model.Model()[1]->Parameters();
+
+  CheckMatrices(parameterA, arma::ones(3 * 3 + 3, 1));
+  CheckMatrices(parameterB, arma::zeros(3 * 4 + 4, 1));
+
+  CheckMatrices(linearA->Parameters(), arma::ones(3 * 3 + 3, 1));
+  CheckMatrices(linearB->Parameters(), arma::zeros(3 * 4 + 4, 1));
+}
+
+/**
+ * Test to see if the FFN code compiles when the Optimizer
+ * doesn't have the MaxIterations() method.
+ */
+TEST_CASE("OptimizerTest", "[FeedForwardNetworkTest]")
+{
+  // Load the dataset.
+  arma::mat trainData;
+  data::Load("thyroid_train.csv", trainData, true);
+
+  arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
+  trainData.shed_row(trainData.n_rows - 1);
+
+  arma::mat testData;
+  data::Load("thyroid_test.csv", testData, true);
+
+  arma::mat testLabels = testData.row(testData.n_rows - 1);
+  testData.shed_row(testData.n_rows - 1);
+
+  FFN<NegativeLogLikelihood<>, RandomInitialization> model;
+  model.Add<Linear>(trainData.n_rows, 8);
+  model.Add<Linear>(8, 3);
+  model.Add<LogSoftMax>();
+
+  ens::DE opt(200, 1000, 0.6, 0.8, 1e-5);
+  model.Train(trainData, trainLabels, opt);
+}
+
+/**
+ * Test to see if an exception is thrown when input with
+ * wrong shape is provided to a FFN.
+ */
+TEST_CASE("FFNCheckInputShapeTest", "[FeedForwardNetworkTest]")
+{
+  // Load the dataset.
+  arma::mat trainData;
+  data::Load("thyroid_train.csv", trainData, true);
+
+  arma::mat trainLabels = trainData.row(trainData.n_rows - 1);
+  trainData.shed_row(trainData.n_rows - 1);
+
+  arma::mat testData;
+  data::Load("thyroid_test.csv", testData, true);
+
+  arma::mat testLabels = testData.row(testData.n_rows - 1);
+  testData.shed_row(testData.n_rows - 1);
+
+  FFN<NegativeLogLikelihood<>, RandomInitialization> model;
+  // Purposely putting wrong input shape so that error is thrown.
+  model.Add<Linear>(trainData.n_rows - 3, 8);
+  model.Add<Linear>(8, 3);
+  model.Add<LogSoftMax>();
+
+  std::string expectedMsg = "FFN<>::Train(): ";
+  expectedMsg += "the first layer of the network expects ";
+  expectedMsg += std::to_string(trainData.n_rows - 3) + " elements, ";
+  expectedMsg += "but the input has " + std::to_string(trainData.n_rows) +
+      " dimensions! ";
+
+  ens::DE opt(200, 1000, 0.6, 0.8, 1e-5);
+
+  REQUIRE_THROWS_AS(model.Train(trainData, trainLabels, opt), std::logic_error);
+}