Revert "Squashed commit of the following:"

This reverts commit 1ea72a60ab.

.ci/linux-steps.yaml

@@ -80,7 +80,6 @@ steps:
     mkdir build && cd build
     if [ "$(binding)" == "go" ]; then
       export GOPATH=$PWD/src/mlpack/bindings/go
-      export GO111MODULE=off
       go get -u -t gonum.org/v1/gonum/...
     fi
     cmake $(CMakeArgs) -DPYTHON_EXECUTABLE=`which python` -DCEREAL_INCLUDE_DIR=/usr/include/ ..

.ci/macos-steps.yaml

@@ -33,7 +33,6 @@ steps:
     mkdir build && cd build
     if [ "$(binding)" == "go" ]; then
       export GOPATH=$PWD/src/mlpack/bindings/go
-      export GO111MODULE=off
       go get -u -t gonum.org/v1/gonum/...
     fi
     if [ "$(binding)" == "python" ]; then

.github/workflows/main.yml

@@ -57,7 +57,6 @@ jobs:
           run: |
             remotes::install_deps(dependencies = TRUE)
             remotes::install_cran("roxygen2")
-            remotes::install_cran("pkgbuild")
           shell: Rscript {0}
 
         - name: CMake

CMake/FindGo.cmake

@@ -14,8 +14,8 @@ if (GO_EXECUTABLE)
        RESULT_VARIABLE RESULT
   )
   if (RESULT EQUAL 0)
-    string(REGEX MATCH "([0-9]+\\.[0-9]+\(\\.[0-9]+\)?)"
-        GO_VERSION_STRING "${GO_VERSION_STRING}")
+    string(REGEX REPLACE ".*([0-9]+\\.[0-9]+\(\\.[0-9]+\)?).*" "\\1"
+        GO_VERSION_STRING ${GO_VERSION_STRING})
   endif()
 endif()
 

CMake/FindGonum.cmake

@@ -4,21 +4,20 @@
 if (GO_EXECUTABLE)
   execute_process(
      COMMAND ${GO_EXECUTABLE} list gonum.org/v1/gonum/mat
-     OUTPUT_VARIABLE GONUM_RAW_STRING
+     OUTPUT_VARIABLE GONUM_VERSION_STRING
      RESULT_VARIABLE RESULT
   )
   if (RESULT EQUAL 0)
+    string(REGEX REPLACE ".*([0-9]+\\.[0-9]+\\.[0-9]+[\n]+).*" "\\1"
+        GONUM_VERSION_STRING ${GONUM_VERSION_STRING})
     string(REGEX REPLACE "\n$" ""
-        GONUM_RAW_STRING ${GONUM_RAW_STRING})
-    if ("${GONUM_RAW_STRING}" STREQUAL "gonum.org/v1/gonum/mat")
-      set(GONUM_FOUND 1)
-    endif()
+        GONUM_VERSION_STRING ${GONUM_VERSION_STRING})
   endif()
 endif()
 
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(
   Gonum
-  REQUIRED_VARS GONUM_FOUND
+  REQUIRED_VARS GONUM_VERSION_STRING
   FAIL_MESSAGE "Gonum not found"
 )

COPYRIGHT.txt

@@ -7,7 +7,7 @@ Source:
 
 Files: *
 Copyright:
-  Copyright 2008-2022, Ryan Curtin <ryan@ratml.org>
+  Copyright 2008-2021, Ryan Curtin <ryan@ratml.org>
   Copyright 2008-2013, Bill March <march@gatech.edu>
   Copyright 2008-2012, Dongryeol Lee <dongryel@cc.gatech.edu>
   Copyright 2008-2013, Nishant Mehta <niche@cc.gatech.edu>

HISTORY.md

@@ -1,7 +1,5 @@
 ### mlpack ?.?.?
 ###### ????-??-??
-  * Fix `Perceptron` to work with cross-validation framework (#3190).
-
   * Migrate from boost tests to Catch2 framework (#2523), (#2584).
 
   * Bump minimum armadillo version from 8.400 to 9.800 (#3043), (#3048).

src/mlpack/bindings/python/copy_artifacts.py

@@ -6,14 +6,18 @@
 # terms of the 3-clause BSD license.  You should have received a copy of the
 # 3-clause BSD license along with mlpack.  If not, see
 # http://www.opensource.org/licenses/BSD-3-Clause for more information.
+import sys
 import sysconfig
 import shutil
 import os
-import glob
 
-# Match any lib.$platform*/mlpack/ directory.
-directory = glob.glob('build/lib.' + sysconfig.get_platform() + '*/mlpack/')[0]
-directory = directory.replace('\\', '/')
+directory = 'build/lib.' + \
+            sysconfig.get_platform() + \
+            '-' + \
+            str(sys.version_info[0]) + \
+            '.' + \
+            str(sys.version_info[1]) + \
+            '/mlpack/'
 
 # Now copy all the files from the directory to the desired location.
 for f in os.listdir(directory):

src/mlpack/bindings/python/is_serializable.hpp

@@ -21,7 +21,7 @@ namespace python {
 template<typename T>
 inline bool IsSerializable(
     util::ParamData& /* d */,
-    const typename std::enable_if<!data::HasSerialize<T>::value>::type* = 0)
+    const typename boost::disable_if<data::HasSerialize<T>>::type* = 0)
 {
   return false;
 }
@@ -29,7 +29,7 @@ inline bool IsSerializable(
 template<typename T>
 inline bool IsSerializable(
     util::ParamData& /* d */,
-    const typename std::enable_if<data::HasSerialize<T>::value>::type* = 0)
+    const typename boost::enable_if<data::HasSerialize<T>>::type* = 0)
 {
   return true;
 }

src/mlpack/core/cereal/CMakeLists.txt

@@ -7,6 +7,8 @@ set(SOURCES
   pair_associative_container.hpp
   pointer_wrapper.hpp
   pointer_vector_wrapper.hpp
+  pointer_variant_wrapper.hpp
+  pointer_vector_variant_wrapper.hpp
   unordered_map.hpp
 )
 

src/mlpack/core/cereal/pointer_variant_wrapper.hpp

@@ -0,0 +1,159 @@
+/**
+ * @file core/cereal/pointer_variant_wrapper.hpp
+ * @author Omar Shrit
+ *
+ * Implementation of a boost::variant wrapper to enable the serialization of
+ * the pointers inside boost variant in cereal
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_CEREAL_POINTER_VARIANT_WRAPPER_HPP
+#define MLPACK_CORE_CEREAL_POINTER_VARIANT_WRAPPER_HPP
+
+#include <cereal/archives/json.hpp>
+#include <cereal/archives/portable_binary.hpp>
+#include <cereal/archives/xml.hpp>
+#include <cereal/types/boost_variant.hpp>
+
+#include <boost/variant.hpp>
+#include <boost/variant/variant_fwd.hpp>
+#include <boost/variant/static_visitor.hpp>
+
+#include "pointer_wrapper.hpp"
+
+namespace cereal {
+
+// Forward declaration.
+template<typename... VariantTypes>
+class PointerVariantWrapper;
+
+/**
+ * Serialize a boost variant in which the variant it self is a raw pointer.
+ * This wrapper will wrap each variant independently by encapsulating each variant
+ * into the PoninterWrapper we have created already.
+ *
+ * @param t A reference to boost variant that holds raw pointer.
+ */
+template<typename... VariantTypes>
+inline PointerVariantWrapper<VariantTypes...>
+make_pointer_variant(boost::variant<VariantTypes...>& t)
+{
+  return PointerVariantWrapper<VariantTypes...>(t);
+}
+
+template<class Archive>
+struct save_visitor : public boost::static_visitor<void>
+{
+  save_visitor(Archive& ar) : ar(ar) {}
+
+  template<class T>
+  void operator()(const T* value) const
+  {
+    ar(CEREAL_POINTER(value));
+  }
+
+  template<typename... Types>
+  void operator()(boost::variant<Types*...>& value) const
+  {
+    ar(make_pointer_variant(value));
+  }
+
+  Archive& ar;
+};
+
+template<typename T>
+struct load_visitor : public boost::static_visitor<void>
+{
+  template<typename Archive, typename VariantType>
+  static void load_impl(Archive& ar, VariantType& variant, std::true_type)
+  {
+    // Note that T will be a pointer type.
+    T loadVariant;
+    ar(CEREAL_POINTER(loadVariant));
+    variant = loadVariant;
+  }
+
+  template<typename Archive, typename VariantType>
+  static void load_impl(Archive& ar, VariantType& value, std::false_type)
+  {
+    // This must be a nested boost::variant.
+    T loadVariant;
+    ar(make_pointer_variant(loadVariant));
+    value = loadVariant;
+  }
+
+  template<typename Archive, typename VariantType>
+  static void load(Archive& ar, VariantType& variant)
+  {
+    // Delegate to the proper load_impl() overload depending on whether T is a
+    // pointer type.  If T is not a pointer type, then we expect it to be a
+    // nested boost::variant.
+    load_impl(ar, variant, typename std::is_pointer<T>::type());
+  }
+};
+
+/**
+ * The objective of this class is to create a wrapper for
+ * boost::variant.
+ * Cereal supports the serialization of boost::variant, but
+ * we need to serialize it if it holds a raw pointers.
+ * This class depeds on the PointerWrapper we have already created in which it is
+ * used to serialize each variant independently
+ */
+template<typename... VariantTypes>
+class PointerVariantWrapper
+{
+ public:
+  PointerVariantWrapper(boost::variant<VariantTypes...>& pointerVar) :
+      pointerVariant(pointerVar)
+  {}
+
+  template<class Archive>
+  void save(Archive& ar) const
+  {
+    // which represents the index in std::variant.
+    int which = pointerVariant.which();
+    ar(CEREAL_NVP(which));
+    save_visitor<Archive> s(ar);
+    boost::apply_visitor(s, pointerVariant);
+  }
+
+  template<class Archive>
+  void load(Archive& ar)
+  {
+    // Load the size of the serialized type.
+    int which;
+    ar(CEREAL_NVP(which));
+
+    // Create function pointers to each overload of load_visitor<T>::load, for
+    // all T in VariantTypes.
+    using LoadFuncType = void(*)(Archive&, boost::variant<VariantTypes...>&);
+    LoadFuncType loadFuncArray[] = { &load_visitor<VariantTypes>::load... };
+
+    if (which >= int(sizeof(loadFuncArray)/sizeof(loadFuncArray[0])))
+      throw std::runtime_error("Invalid 'which' selector when"
+          "deserializing boost::variant");
+
+    loadFuncArray[which](ar, pointerVariant);
+  }
+
+ private:
+  boost::variant<VariantTypes...>& pointerVariant;
+};
+
+/**
+ * Cereal does not support the serialization of raw pointer.
+ * This macro enable developers to serialize boost::variant that holds raw
+ * pointers by using the above PointerVariantWrapper class which replace the
+ * internal raw pointers by smart pointer internally.
+ *
+ * @param T boost::variant that holds raw pointer to be serialized.
+ */
+#define CEREAL_VARIANT_POINTER(T) cereal::make_pointer_variant(T)
+
+} // namespace cereal
+
+#endif // CEREAL_POINTER_VARIANT_WRAPPER_HPP

src/mlpack/core/cereal/pointer_vector_variant_wrapper.hpp

@@ -0,0 +1,97 @@
+/**
+ * @file core/cereal/pointer_vector_variant_wrapper.hpp
+ * @author Omar Shrit
+ *
+ * Implementation of a boost::variant wrapper to enable the serialization of
+ * the pointers inside boost variant in cereal
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_CEREAL_POINTER_VECTOR_VARIANT_WRAPPER_HPP
+#define MLPACK_CORE_CEREAL_POINTER_VECTOR_VARIANT_WRAPPER_HPP
+
+#include "pointer_wrapper.hpp"
+#include "pointer_variant_wrapper.hpp"
+#include "pointer_vector_wrapper.hpp"
+
+namespace cereal {
+
+// Forward declaration
+template<typename... VariantTypes>
+class PointerVectorVariantWrapper;
+
+/**
+ * Serialize a std::vector of boost variants in which the variant in each boost
+ * variant is a raw pointer.
+ * This wrapper will wrap each boost variant independently by encapsulating each
+ * boost variant into the PoninterVariantWrapper we have created already.
+ *
+ * @param t A reference to a vector of boost variants that holds raw pointer.
+ */
+template<typename... VariantTypes>
+inline PointerVectorVariantWrapper<VariantTypes...>
+make_vector_pointer_variant(std::vector<boost::variant<VariantTypes...>>& t)
+{
+  return PointerVectorVariantWrapper<VariantTypes...>(t);
+}
+
+/**
+ * The objective of this class is to create a wrapper for
+ * a vector of boost::variant that holds pointer.
+ * Cereal supports the serialization of boost::variant, but 
+ * we need to serialize it if it holds a vector of boost::variant that holds a
+ * pointers.
+ */
+template<typename... VariantTypes>
+class PointerVectorVariantWrapper
+{
+ public:
+  PointerVectorVariantWrapper(
+      std::vector<boost::variant<VariantTypes...>>& vecPointerVar)
+      : vectorPointerVariant(vecPointerVar)
+  {}
+
+  template<class Archive>
+  void save(Archive& ar) const
+  {
+    size_t vecSize = vectorPointerVariant.size();
+    ar(CEREAL_NVP(vecSize));
+    for (size_t i = 0; i < vectorPointerVariant.size(); ++i)
+    {
+      ar(CEREAL_VARIANT_POINTER(vectorPointerVariant.at(i)));
+    }
+  }
+
+  template<class Archive>
+  void load(Archive& ar)
+  {
+    size_t vecSize = 0;
+    ar(CEREAL_NVP(vecSize));
+    vectorPointerVariant.resize(vecSize);
+    for (size_t i = 0; i < vectorPointerVariant.size(); ++i)
+    {
+      ar(CEREAL_VARIANT_POINTER(vectorPointerVariant.at(i)));
+    }
+  }
+
+ private:
+  std::vector<boost::variant<VariantTypes...>>& vectorPointerVariant;
+};
+
+/**
+ * Cereal does not support the serialization of raw pointer.
+ * This macro enable developers to serialize a std vector that holds boost::variants
+ * that holds raw pointers by using the above PointerVectorVariantWrapper class
+ * which replace the internal raw pointers by smart pointer internally.
+ *
+ * @param T std::vector<boost::variant> that holds raw pointer to be serialized.
+ */
+#define CEREAL_VECTOR_VARIANT_POINTER(T) cereal::make_vector_pointer_variant(T)
+
+} // namespace cereal
+
+#endif // CEREAL_POINTER_VECTOR_VARIANT_WRAPPER_HPP
+

src/mlpack/core/util/size_checks.hpp

@@ -26,27 +26,18 @@ namespace util {
  *     error generation.
  * @param addInfo Name to use for labels for precise error generation. Default
  *     is "labels"; for example, "weights" could also be used.
- * @param isDataTranspose Bool parameter which can be set true to transpose data
- *     before size-check. Default is false.
- * @param isLabelTranspose Bool parameter which can be set true to transpose label
- *     before size-check. Default is false.
  */
 template<typename DataType, typename LabelsType>
 inline void CheckSameSizes(const DataType& data,
                            const LabelsType& label,
                            const std::string& callerDescription,
-                           const std::string& addInfo = "labels",
-                           const bool& isDataTranspose = false,
-                           const bool& isLabelTranspose = false)
-{ 
-  const size_t dataPoints = (isDataTranspose == true) ? data.n_rows : data.n_cols;
-  const size_t labelPoints = (isLabelTranspose == true) ? label.n_rows : label.n_cols;
-  
-  if (dataPoints != labelPoints)
+                           const std::string& addInfo = "labels")
+{
+  if (data.n_cols != label.n_cols)
   {
     std::ostringstream oss;
-    oss << callerDescription << ": number of points (" << dataPoints << ") "
-        << "does not match number of " << addInfo << " (" << labelPoints
+    oss << callerDescription << ": number of points (" << data.n_cols << ") "
+        << "does not match number of " << addInfo << " (" << label.n_cols
         << ")!" << std::endl;
     throw std::invalid_argument(oss.str());
   }

src/mlpack/methods/ann/CMakeLists.txt

@@ -3,17 +3,24 @@
 set(SOURCES
   ffn.hpp
   ffn_impl.hpp
-  forward_decls.hpp
-  make_alias.hpp
   rnn.hpp
   rnn_impl.hpp
+  brnn.hpp
+  brnn_impl.hpp
+  layer_names.hpp
 )
 
+add_subdirectory(visitor)
+add_subdirectory(activation_functions)
 add_subdirectory(init_rules)
 add_subdirectory(layer)
 add_subdirectory(loss_functions)
 add_subdirectory(convolution_rules)
+add_subdirectory(gan)
+add_subdirectory(rbm)
+add_subdirectory(augmented)
 add_subdirectory(regularizer)
+add_subdirectory(util)
 
 # Add directory name to sources.
 set(DIR_SRCS)

src/mlpack/methods/ann/brnn.hpp

@@ -24,6 +24,7 @@
 #include "init_rules/network_init.hpp"
 #include <mlpack/methods/ann/layer/layer_types.hpp>
 #include <mlpack/methods/ann/layer/layer.hpp>
+#include <mlpack/methods/ann/layer/layer_traits.hpp>
 #include <mlpack/methods/ann/init_rules/random_init.hpp>
 
 #include <ensmallen.hpp>

src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp

@@ -57,15 +57,9 @@ class NaiveConvolution
               const size_t dilationW = 1,
               const size_t dilationH = 1)
   {
-    // Compute the output size.  The filterRows and filterCols computation must
-    // take into account the fact that dilation only adds rows or columns
-    // *between* filter elements.  So, e.g., a dilation of 2 on a kernel size of
-    // 3x3 means an effective kernel size of 5x5, *not* 6x6.
-    const size_t filterRows = filter.n_rows * dilationH - (dilationH - 1);
-    const size_t filterCols = filter.n_cols * dilationW - (dilationW - 1);
-    const size_t outputRows = (input.n_rows - filterRows + dH) / dH;
-    const size_t outputCols = (input.n_cols - filterCols + dW) / dW;
-    output.zeros(outputRows, outputCols);
+    output = arma::zeros<arma::Mat<eT> >(
+        (input.n_rows - (filter.n_rows - 1) * dilationW - 1) / dW + 1,
+        (input.n_cols - (filter.n_cols - 1) * dilationH -  1) / dH + 1);
 
     // It seems to be about 3.5 times faster to use pointers instead of
     // filter(ki, kj) * input(leftInput + ki, topInput + kj) and output(i, j).
@@ -109,22 +103,37 @@ class NaiveConvolution
               const size_t dilationW = 1,
               const size_t dilationH = 1)
   {
-    // First, compute the necessary padding for the full convolution.  It is
-    // possible that this might be an overestimate.  Note that these variables
-    // only hold the padding on one side of the input.
-    const size_t filterRows = filter.n_rows * dilationH - (dilationH - 1);
-    const size_t filterCols = filter.n_cols * dilationW - (dilationW - 1);
-    const size_t paddingRows = filterRows - 1;
-    const size_t paddingCols = filterCols - 1;
+    size_t outputRows = (input.n_rows - 1) * dW + 2 * (filter.n_rows - 1)
+        * dilationW + 1;
+    size_t outputCols = (input.n_cols - 1) * dH + 2 * (filter.n_cols - 1)
+        * dilationH + 1;
+
+    for (size_t i = 0; i < dW; ++i)
+    {
+      if (((((i + outputRows - 2 * (filter.n_rows - 1) * dilationW - 1) % dW)
+          + dW) % dW) == i){
+        outputRows += i;
+        break;
+      }
+    }
+    for (size_t i = 0; i < dH; ++i)
+    {
+      if (((((i + outputCols - 2 * (filter.n_cols - 1) * dilationH - 1) % dH)
+          + dH) % dH) == i){
+        outputCols += i;
+        break;
+      }
+    }
 
     // Pad filter and input to the working output shape.
-    arma::Mat<eT> inputPadded(input.n_rows + 2 * paddingRows,
-        input.n_cols + 2 * paddingCols, arma::fill::zeros);
-    inputPadded.submat(paddingRows, paddingCols, paddingRows + input.n_rows - 1,
-        paddingCols + input.n_cols - 1) = input;
+    arma::Mat<eT> inputPadded = arma::zeros<arma::Mat<eT> >(outputRows,
+        outputCols);
+    inputPadded.submat((filter.n_rows - 1) * dilationW, (filter.n_cols - 1)
+        * dilationH, (filter.n_rows - 1) * dilationW + input.n_rows - 1,
+        (filter.n_cols - 1) * dilationH + input.n_cols - 1) = input;
 
     NaiveConvolution<ValidConvolution>::Convolution(inputPadded, filter,
-        output, dW, dH, dilationW, dilationH);
+        output, 1, 1, dilationW, dilationH);
   }
 
   /*

src/mlpack/methods/ann/ffn.hpp

@@ -15,49 +15,46 @@
 
 #include <mlpack/prereqs.hpp>
 
-#include "forward_decls.hpp"
+#include "visitor/delete_visitor.hpp"
+#include "visitor/delta_visitor.hpp"
+#include "visitor/output_height_visitor.hpp"
+#include "visitor/output_parameter_visitor.hpp"
+#include "visitor/output_width_visitor.hpp"
+#include "visitor/reset_visitor.hpp"
+#include "visitor/weight_size_visitor.hpp"
+#include "visitor/copy_visitor.hpp"
+#include "visitor/loss_visitor.hpp"
+
 #include "init_rules/network_init.hpp"
 
+#include <mlpack/methods/ann/layer/layer_types.hpp>
 #include <mlpack/methods/ann/layer/layer.hpp>
-#include <mlpack/methods/ann/layer/multi_layer.hpp>
 #include <mlpack/methods/ann/init_rules/random_init.hpp>
-#include <mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp>
+#include <mlpack/methods/ann/layer/layer_traits.hpp>
 #include <ensmallen.hpp>
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
 /**
- * Implementation of a standard feed forward network.  Any layer that inherits
- * from the base `Layer` class can be added to this model.  For recursive neural
- * networks, see the `RNN` class.
- *
- * In general, a network can be created by using the `Add()` method to add
- * layers to the network.  Then, training can be performed with `Train()`, and
- * data points can be passed through the trained network with `Predict()`.
- *
- * Although the actual types passed as input will be matrix objects with one
- * data point per column, each data point can be a tensor of arbitrary shape.
- * If data points are not 1-dimensional vectors, then set the shape of the input
- * with `InputDimensions()` before calling `Train()`.
- *
- * More granular functionality is available with `Forward()`, Backward()`, and
- * `Evaluate()`, or even by accessing the individual layers directly with
- * `Network()`.
+ * Implementation of a standard feed forward network.
  *
  * @tparam OutputLayerType The output layer type used to evaluate the network.
  * @tparam InitializationRuleType Rule used to initialize the weight matrix.
- * @tparam MatType Type of matrix to be given as input to the network.
- * @tparam MatType Type of matrix to be produced as output from the last
- *     layer.
+ * @tparam CustomLayers Any set of custom layers that could be a part of the
+ *         feed forward network.
  */
 template<
-    typename OutputLayerType = NegativeLogLikelihood,
-    typename InitializationRuleType = RandomInitialization,
-    typename MatType = arma::mat>
+  typename OutputLayerType = NegativeLogLikelihood<>,
+  typename InitializationRuleType = RandomInitialization,
+  typename... CustomLayers
+>
 class FFN
 {
  public:
+  //! Convenience typedef for the internal model construction.
+  using NetworkType = FFN<OutputLayerType, InitializationRuleType>;
+
   /**
    * Create the FFN object.
    *
@@ -75,73 +72,56 @@ class FFN
       InitializationRuleType initializeRule = InitializationRuleType());
 
   //! Copy constructor.
-  FFN(const FFN& other);
+  FFN(const FFN&);
+
   //! Move constructor.
-  FFN(FFN&& other);
-  //! Copy operator.
-  FFN& operator=(const FFN& other);
-  //! Move assignment operator.
-  FFN& operator=(FFN&& other);
+  FFN(FFN&&);
+
+  //! Copy/move assignment operator.
+  FFN& operator = (FFN);
+
+  //! Destructor to release allocated memory.
+  ~FFN();
 
   /**
-   * Add a new layer to the model.
+   * Check if the optimizer has MaxIterations() parameter, if it does
+   * then check if it's value is less than the number of datapoints
+   * in the dataset.
    *
-   * @param args The layer parameter.
+   * @tparam OptimizerType Type of optimizer to use to train the model.
+   * @param optimizer optimizer used in the training process.
+   * @param samples Number of datapoints in the dataset.
    */
-  template <typename LayerType, typename... Args>
-  void Add(Args... args)
-  {
-    network.template Add<LayerType>(args...);
-    inputDimensionsAreSet = false;
-  }
+  template<typename OptimizerType>
+  typename std::enable_if<
+      HasMaxIterations<OptimizerType, size_t&(OptimizerType::*)()>
+      ::value, void>::type
+  WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const;
 
   /**
-   * Add a new layer to the model.  Note that any trainable weights of this
-   * layer will be reset!  (Any constant parameters are kept.)
+   * Check if the optimizer has MaxIterations() parameter, if it
+   * doesn't then simply return from the function.
    *
-   * @param layer The Layer to be added to the model.
+   * @tparam OptimizerType Type of optimizer to use to train the model.
+   * @param optimizer optimizer used in the training process.
+   * @param samples Number of datapoints in the dataset.
    */
-  void Add(Layer<MatType>* layer)
-  {
-    network.Add(layer);
-    inputDimensionsAreSet = false;
-  }
-
-  //! Get the layers of the network.
-  const std::vector<Layer<MatType>*>& Network() const
-  {
-    return network.Network();
-  }
-
-  /**
-   * Modify the network model.  Be careful!  If you change the structure of the
-   * network or parameters for layers, its state may become invalid, and the
-   * next time it is used for any operation the parameters will be reset.
-   *
-   * Don't add any layers like this; use `Add()` instead.
-   */
-  std::vector<Layer<MatType>*>& Network()
-  {
-    // We can no longer make any assumptions... the user may change anything.
-    inputDimensionsAreSet = false;
-    layerMemoryIsSet = false;
-
-    return network.Network();
-  }
+  template<typename OptimizerType>
+  typename std::enable_if<
+      !HasMaxIterations<OptimizerType, size_t&(OptimizerType::*)()>
+      ::value, void>::type
+  WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const;
 
   /**
    * Train the feedforward network on the given input data using the given
    * optimizer.
    *
-   * If no parameters have ever been set (e.g. if `Parameters()` is an empty
-   * matrix), or if the parameters' size does not match the number of weights
-   * needed for the current input size (as given by `predictors` and optionally
-   * set further by `InputDimensions()`), then the network will be initialized
-   * using `InitializeRuleType`.
+   * This will use the existing model parameters as a starting point for the
+   * optimization. If this is not what you want, then you should access the
+   * parameters vector directly with Parameters() and modify it as desired.
    *
-   * If parameters are the right size for the given `predictors` and
-   * `InputDimensions()`, then the existing parameters will be used as a
-   * starting point.  (If you want to reinitialize, first call `Reset()`.)
+   * If you want to pass in a parameter and discard the original parameter
+   * object, be sure to use std::move to avoid unnecessary copy.
    *
    * @tparam OptimizerType Type of optimizer to use to train the model.
    * @tparam CallbackTypes Types of Callback Functions.
@@ -153,25 +133,22 @@ class FFN
    * @return The final objective of the trained model (NaN or Inf on error).
    */
   template<typename OptimizerType, typename... CallbackTypes>
-  typename MatType::elem_type Train(MatType predictors,
-                                    MatType responses,
-                                    OptimizerType& optimizer,
-                                    CallbackTypes&&... callbacks);
+  double Train(arma::mat predictors,
+               arma::mat responses,
+               OptimizerType& optimizer,
+               CallbackTypes&&... callbacks);
 
   /**
    * Train the feedforward network on the given input data. By default, the
    * RMSProp optimization algorithm is used, but others can be specified
    * (such as ens::SGD).
    *
-   * If no parameters have ever been set (e.g. if `Parameters()` is an empty
-   * matrix), or if the parameters' size does not match the number of weights
-   * needed for the current input size (as given by `predictors` and optionally
-   * set further by `InputDimensions()`), then the network will be initialized
-   * using `InitializeRuleType`.
+   * This will use the existing model parameters as a starting point for the
+   * optimization. If this is not what you want, then you should access the
+   * parameters vector directly with Parameters() and modify it as desired.
    *
-   * If parameters are the right size for the given `predictors` and
-   * `InputDimensions()`, then the existing parameters will be used as a
-   * starting point.  (If you want to reinitialize, first call `Reset()`.)
+   * If you want to pass in a parameter and discard the original parameter
+   * object, be sure to use std::move to avoid unnecessary copy.
    *
    * @tparam OptimizerType Type of optimizer to use to train the model.
    * @param predictors Input training variables.
@@ -182,123 +159,22 @@ class FFN
    * @return The final objective of the trained model (NaN or Inf on error).
    */
   template<typename OptimizerType = ens::RMSProp, typename... CallbackTypes>
-  typename MatType::elem_type Train(MatType predictors,
-                                    MatType responses,
-                                    CallbackTypes&&... callbacks);
+  double Train(arma::mat predictors,
+               arma::mat responses,
+               CallbackTypes&&... callbacks);
 
   /**
-   * Predict the responses to a given set of predictors. The responses will be
-   * the output of the output layer when `predictors` is passed through the
-   * whole network (`OutputLayerType`).
+   * Predict the responses to a given set of predictors. The responses will
+   * reflect the output of the given output layer as returned by the
+   * output layer function.
+   *
+   * If you want to pass in a parameter and discard the original parameter
+   * object, be sure to use std::move to avoid unnecessary copy.
    *
    * @param predictors Input predictors.
    * @param results Matrix to put output predictions of responses into.
-   * @param batchSize Batch size to use for prediction.
    */
-  void Predict(MatType predictors,
-               MatType& results,
-               const size_t batchSize = 128);
-
-  // Return the number of weights in the model.
-  size_t WeightSize();
-
-  /**
-   * Set the logical dimensions of the input.  `Train()` and `Predict()` expect
-   * data to be passed such that one point corresponds to one column, but this
-   * data is allowed to be an arbitrary higher-order tensor.
-   *
-   * So, if the input is meant to be 28x28x3 images, then the
-   * input data to `Train()` or `Predict()` should have 28*28*3 = 2352 rows, and
-   * `InputDimensions()` should be set to `{ 28, 28, 3 }`.  Then, the layers of
-   * the network will interpret each input point as a 3-dimensional image
-   * instead of a 1-dimensional vector.
-   *
-   * If `InputDimensions()` is left unset before training, the data will be
-   * assumed to be a 1-dimensional vector.
-   */
-  std::vector<size_t>& InputDimensions()
-  {
-    // The user may change the input dimensions, so we will have to propagate
-    // these changes to the network.
-    inputDimensionsAreSet = false;
-    return inputDimensions;
-  }
-  //! Get the logical dimensions of the input.
-  const std::vector<size_t>& InputDimensions() const { return inputDimensions; }
-
-  //! Return the current set of weights.  These are linearized: this contains
-  //! the weights of every layer.
-  const MatType& Parameters() const { return parameters; }
-  //! Modify the current set of weights.  These are linearized: this contains
-  //! the weights of every layer.  Be careful!  If you change the shape of
-  //! `parameters` to something incorrect, it may be re-initialized the next
-  //! time a forward pass is done.
-  MatType& Parameters() { return parameters; }
-
-  /**
-   * Reset the stored data of the network entirely.  This resets all weights of
-   * each layer using `InitializationRuleType`, and prepares the network to
-   * accept a (flat 1-d) input size of `inputDimensionality` (if passed), or
-   * whatever input size has been set with `InputDimensions()`.
-   *
-   * This also resets the mode of the network to prediction mode (not training
-   * mode).  See `SetNetworkMode()` for more information.
-   */
-  void Reset(const size_t inputDimensionality = 0);
-
-  /**
-   * Set all the layers in the network to training mode, if `training` is
-   * `true`, or set all the layers in the network to testing mode, if `training`
-   * is `false`.
-   */
-  void SetNetworkMode(const bool training);
-
-  /**
-   * Perform a manual forward pass of the data.
-   *
-   * `Forward()` and `Backward()` should be used as a pair, and they are
-   * designed mainly for advanced users. You should try to use `Predict()` and
-   * `Train()`, if you can.
-   *
-   * @param inputs The input data.
-   * @param results The predicted results.
-   */
-  void Forward(const MatType& inputs, MatType& results);
-
-  /**
-   * Perform a manual partial forward pass of the data.
-   *
-   * This function is meant for the cases when users require a forward pass only
-   * through certain layers and not the entire network.  `Forward()` and
-   * `Backward()` should be used as a pair, and they are designed mainly for
-   * advanced users. You should try to use `Predict()` and `Train()`, if you
-   * can.
-   *
-   * @param inputs The input data for the specified first layer.
-   * @param results The predicted results from the specified last layer.
-   * @param begin The index of the first layer.
-   * @param end The index of the last layer.
-   */
-  void Forward(const MatType& inputs,
-               MatType& results,
-               const size_t begin,
-               const size_t end);
-
-  /**
-   * Perform a manual backward pass of the data.
-   *
-   * `Forward()` and `Backward()` should be used as a pair, and they are
-   * designed mainly for advanced users. You should try to use `Predict()` and
-   * `Train()` instead, if you can.
-   *
-   * @param inputs Inputs of current pass.
-   * @param targets The training target.
-   * @param gradients Computed gradients.
-   * @return Training error of the current pass.
-   */
-  typename MatType::elem_type Backward(const MatType& inputs,
-                                       const MatType& targets,
-                                       MatType& gradients);
+  void Predict(arma::mat predictors, arma::mat& results);
 
   /**
    * Evaluate the feedforward network with the given predictors and responses.
@@ -307,38 +183,41 @@ class FFN
    * @param predictors Input variables.
    * @param responses Target outputs for input variables.
    */
-  typename MatType::elem_type Evaluate(const MatType& predictors,
-                                       const MatType& responses);
-
-  //! Serialize the model.
-  template<typename Archive>
-  void serialize(Archive& ar, const uint32_t /* version */);
-
-  //
-  // Only ensmallen utility functions for training are found below here.
-  // They aren't generally useful otherwise.
-  //
+  template<typename PredictorsType, typename ResponsesType>
+  double Evaluate(const PredictorsType& predictors,
+                  const ResponsesType& responses);
 
   /**
-   * Note: this function is implemented so that it can be used by ensmallen's
-   * optimizers.  It's not generally meant to be used otherwise.
-   *
-   * Evaluate the feedforward network with the given parameters.
+   * Evaluate the feedforward network with the given parameters. This function
+   * is usually called by the optimizer to train the model.
    *
    * @param parameters Matrix model parameters.
    */
-  typename MatType::elem_type Evaluate(const MatType& parameters);
+  double Evaluate(const arma::mat& parameters);
 
-  /**
-   * Note: this function is implemented so that it can be used by ensmallen's
-   * optimizers.  It's not generally meant to be used otherwise.
-   *
+   /**
    * Evaluate the feedforward network with the given parameters, but using only
    * a number of data points. This is useful for optimizers such as SGD, which
    * require a separable objective function.
    *
-   * Note that the network may return different results depending on the mode it
-   * is in (see `SetNetworkMode()`).
+   * @param parameters Matrix model parameters.
+   * @param begin Index of the starting point to use for objective function
+   *        evaluation.
+   * @param batchSize Number of points to be passed at a time to use for
+   *        objective function evaluation.
+   * @param deterministic Whether or not to train or test the model. Note some
+   *        layer act differently in training or testing mode.
+   */
+  double Evaluate(const arma::mat& parameters,
+                  const size_t begin,
+                  const size_t batchSize,
+                  const bool deterministic);
+
+   /**
+   * Evaluate the feedforward network with the given parameters, but using only
+   * a number of data points. This is useful for optimizers such as SGD, which
+   * require a separable objective function. This just calls the overload of
+   * Evaluate() with deterministic = true.
    *
    * @param parameters Matrix model parameters.
    * @param begin Index of the starting point to use for objective function
@@ -346,14 +225,11 @@ class FFN
    * @param batchSize Number of points to be passed at a time to use for
    *        objective function evaluation.
    */
-  typename MatType::elem_type Evaluate(const MatType& parameters,
-                                       const size_t begin,
-                                       const size_t batchSize);
+  double Evaluate(const arma::mat& parameters,
+                  const size_t begin,
+                  const size_t batchSize);
 
   /**
-   * Note: this function is implemented so that it can be used by ensmallen's
-   * optimizers.  It's not generally meant to be used otherwise.
-   *
    * Evaluate the feedforward network with the given parameters.
    * This function is usually called by the optimizer to train the model.
    * This just calls the overload of EvaluateWithGradient() with batchSize = 1.
@@ -361,13 +237,10 @@ class FFN
    * @param parameters Matrix model parameters.
    * @param gradient Matrix to output gradient into.
    */
-  typename MatType::elem_type EvaluateWithGradient(const MatType& parameters,
-                                                   MatType& gradient);
+  template<typename GradType>
+  double EvaluateWithGradient(const arma::mat& parameters, GradType& gradient);
 
-  /**
-   * Note: this function is implemented so that it can be used by ensmallen's
-   * optimizers.  It's not generally meant to be used otherwise.
-   *
+   /**
    * Evaluate the feedforward network with the given parameters, but using only
    * a number of data points. This is useful for optimizers such as SGD, which
    * require a separable objective function.
@@ -379,15 +252,13 @@ class FFN
    * @param batchSize Number of points to be passed at a time to use for
    *        objective function evaluation.
    */
-  typename MatType::elem_type EvaluateWithGradient(const MatType& parameters,
-                                                   const size_t begin,
-                                                   MatType& gradient,
-                                                   const size_t batchSize);
+  template<typename GradType>
+  double EvaluateWithGradient(const arma::mat& parameters,
+                              const size_t begin,
+                              GradType& gradient,
+                              const size_t batchSize);
 
   /**
-   * Note: this function is implemented so that it can be used by ensmallen's
-   * optimizers.  It's not generally meant to be used otherwise.
-   *
    * Evaluate the gradient of the feedforward network with the given parameters,
    * and with respect to only a number of points in the dataset. This is useful
    * for optimizers such as SGD, which require a separable objective function.
@@ -399,156 +270,253 @@ class FFN
    * @param batchSize Number of points to be processed as a batch for objective
    *        function gradient evaluation.
    */
-  void Gradient(const MatType& parameters,
+  void Gradient(const arma::mat& parameters,
                 const size_t begin,
-                MatType& gradient,
+                arma::mat& gradient,
                 const size_t batchSize);
 
   /**
-   * Note: this function is implemented so that it can be used by ensmallen's
-   * optimizers.  It's not generally meant to be used otherwise.
-   *
-   * Return the number of separable functions (the number of predictor points).
-   */
-  size_t NumFunctions() const { return responses.n_cols; }
-
-  /**
-   * Note: this function is implemented so that it can be used by ensmallen's
-   * optimizers.  It's not generally meant to be used otherwise.
-   *
-   * Shuffle the order of function visitation.  (This is equivalent to shuffling
-   * the dataset during training.)
+   * Shuffle the order of function visitation. This may be called by the
+   * optimizer.
    */
   void Shuffle();
 
-  /**
-   * Prepare the network for training on the given data.
+  /*
+   * Add a new module to the model.
    *
-   * This function won't actually trigger the training process, and is
-   * generally only useful internally.
+   * @param args The layer parameter.
+   */
+  template <class LayerType, class... Args>
+  void Add(Args... args) { network.push_back(new LayerType(args...)); }
+
+  /*
+   * Add a new module to the model.
+   *
+   * @param layer The Layer to be added to the model.
+   */
+  void Add(LayerTypes<CustomLayers...> layer) { network.push_back(layer); }
+
+  //! Get the network model.
+  const std::vector<LayerTypes<CustomLayers...> >& Model() const
+  {
+    return network;
+  }
+  //! Modify the network model.  Be careful!  If you change the structure of the
+  //! network or parameters for layers, its state may become invalid, so be sure
+  //! to call ResetParameters() afterwards.
+  std::vector<LayerTypes<CustomLayers...> >& Model() { return network; }
+
+  //! Return the number of separable functions (the number of predictor points).
+  size_t NumFunctions() const { return numFunctions; }
+
+  //! Return the initial point for the optimization.
+  const arma::mat& Parameters() const { return parameter; }
+  //! Modify the initial point for the optimization.
+  arma::mat& Parameters() { return parameter; }
+
+  //! Get the matrix of responses to the input data points.
+  const arma::mat& Responses() const { return responses; }
+  //! Modify the matrix of responses to the input data points.
+  arma::mat& Responses() { return responses; }
+
+  //! Get the matrix of data points (predictors).
+  const arma::mat& Predictors() const { return predictors; }
+  //! Modify the matrix of data points (predictors).
+  arma::mat& Predictors() { return predictors; }
+
+  /**
+   * Reset the module infomration (weights/parameters).
+   */
+  void ResetParameters();
+
+  //! Serialize the model.
+  template<typename Archive>
+  void serialize(Archive& ar, const uint32_t /* version */);
+
+  /**
+   * Perform the forward pass of the data in real batch mode.
+   *
+   * Forward and Backward should be used as a pair, and they are designed mainly
+   * for advanced users. User should try to use Predict and Train unless those
+   * two functions can't satisfy some special requirements.
+   *
+   * @param inputs The input data.
+   * @param results The predicted results.
+   */
+  template<typename PredictorsType, typename ResponsesType>
+  void Forward(const PredictorsType& inputs, ResponsesType& results);
+
+  /**
+   * Perform a partial forward pass of the data.
+   *
+   * This function is meant for the cases when users require a forward pass only
+   * through certain layers and not the entire network.
+   *
+   * @param inputs The input data for the specified first layer.
+   * @param results The predicted results from the specified last layer.
+   * @param begin The index of the first layer.
+   * @param end The index of the last layer.
+   */
+  template<typename PredictorsType, typename ResponsesType>
+  void Forward(const PredictorsType& inputs ,
+               ResponsesType& results,
+               const size_t begin,
+               const size_t end);
+
+  /**
+   * Perform the backward pass of the data in real batch mode.
+   *
+   * Forward and Backward should be used as a pair, and they are designed mainly
+   * for advanced users. User should try to use Predict and Train unless those
+   * two functions can't satisfy some special requirements.
+   *
+   * @param inputs Inputs of current pass.
+   * @param targets The training target.
+   * @param gradients Computed gradients.
+   * @return Training error of the current pass.
+   */
+  template<typename PredictorsType,
+           typename TargetsType,
+           typename GradientsType>
+  double Backward(const PredictorsType& inputs,
+                  const TargetsType& targets,
+                  GradientsType& gradients);
+
+ private:
+  // Helper functions.
+  /**
+   * The Forward algorithm (part of the Forward-Backward algorithm).  Computes
+   * forward probabilities for each module.
+   *
+   * @param input Data sequence to compute probabilities for.
+   */
+  template<typename InputType>
+  void Forward(const InputType& input);
+
+  /**
+   * Prepare the network for the given data.
+   * This function won't actually trigger training process.
    *
    * @param predictors Input data variables.
    * @param responses Outputs results from input data variables.
    */
-  void ResetData(MatType predictors, MatType responses);
-
- private:
-  // Helper functions.
-
-  //! Use the InitializationPolicy to initialize all the weights in the network.
-  void InitializeWeights();
-
-  //! Make the memory of each layer point to the right place, by calling
-  //! SetWeightPtr() on each layer.
-  void SetLayerMemory();
+  void ResetData(arma::mat predictors, arma::mat responses);
 
   /**
-   * Ensure that all the locally-cached information about the network is valid,
-   * all parameter memory is initialized, and we can make forward and backward
-   * passes.
+   * The Backward algorithm (part of the Forward-Backward algorithm). Computes
+   * backward pass for module.
+   */
+  void Backward();
+
+  /**
+   * Iterate through all layer modules and update the the gradient using the
+   * layer defined optimizer.
+   */
+  template<typename InputType>
+  void Gradient(const InputType& input);
+
+  /**
+   * Reset the module status by setting the current deterministic parameter
+   * for all modules that implement the Deterministic function.
+   */
+  void ResetDeterministic();
+
+  /**
+   * Reset the gradient for all modules that implement the Gradient function.
+   */
+  void ResetGradients(arma::mat& gradient);
+
+  /**
+   * Swap the content of this network with given network.
    *
-   * @param functionName Name of function to use if an exception is thrown.
-   * @param inputDimensionality Given dimensionality of the input data.
-   * @param setMode If true, the mode of the network will be set to the
-   *     parameter given in `training`.  Otherwise the mode of the network is
-   *     left unmodified.
-   * @param training Mode to set the network to; `true` indicates the network
-   *     should be set to training mode; `false` indicates testing mode.
+   * @param network Desired source network.
    */
-  void CheckNetwork(const std::string& functionName,
-                    const size_t inputDimensionality,
-                    const bool setMode = false,
-                    const bool training = false);
+  void Swap(FFN& network);
 
-  /**
-   * Set the input and output dimensions of each layer in the network correctly.
-   * The size of the input is taken, in case `inputDimensions` has not been set
-   * otherwise (e.g. via `InputDimensions()`).  If `InputDimensions()` is not
-   * empty, then `inputDimensionality` is ignored.
-   */
-  void UpdateDimensions(const std::string& functionName,
-                        const size_t inputDimensionality = 0);
-
-  /**
-   * Check if the optimizer has MaxIterations() parameter, if it does then check
-   * if its value is less than the number of datapoints in the dataset.
-   *
-   * @tparam OptimizerType Type of optimizer to use to train the model.
-   * @param optimizer optimizer used in the training process.
-   * @param samples Number of datapoints in the dataset.
-   */
-  template<typename OptimizerType>
-  typename std::enable_if<
-      ens::traits::HasMaxIterationsSignature<OptimizerType>::value, void
-  >::type
-  WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const;
-
-  /**
-   * Check if the optimizer has MaxIterations() parameter; if it doesn't then
-   * simply return from the function.
-   *
-   * @tparam OptimizerType Type of optimizer to use to train the model.
-   * @param optimizer optimizer used in the training process.
-   * @param samples Number of datapoints in the dataset.
-   */
-  template<typename OptimizerType>
-  typename std::enable_if<
-      !ens::traits::HasMaxIterationsSignature<OptimizerType>::value, void
-  >::type
-  WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const;
-
-  //! Instantiated output layer used to evaluate the network.
+  //! Instantiated outputlayer used to evaluate the network.
   OutputLayerType outputLayer;
 
   //! Instantiated InitializationRule object for initializing the network
   //! parameter.
   InitializationRuleType initializeRule;
 
-  //! All of the network is stored inside this multilayer.
-  MultiLayer<MatType> network;
+  //! The input width.
+  size_t width;
 
-  /**
-   * Matrix of (trainable) parameters.  Each weight here corresponds to a layer,
-   * and each layer's `parameters` member is an alias pointing to parameters in
-   * this matrix.
-   *
-   * Note: although each layer may have its own MatType and MatType,
-   * ensmallen optimization requires everything to be stored in one matrix
-   * object, so we have chosen MatType.  This could be made more flexible
-   * with a "wrapper" class implementing the Armadillo API.
-   */
-  MatType parameters;
+  //! The input height.
+  size_t height;
 
-  //! Dimensions of input data.
-  std::vector<size_t> inputDimensions;
+  //! Indicator if we already trained the model.
+  bool reset;
 
-  //! The matrix of data points (predictors).  This member is empty, except
-  //! during training---we must store a local copy of the training data since
-  //! the ensmallen optimizer will not provide training data.
-  MatType predictors;
+  //! Locally-stored model modules.
+  std::vector<LayerTypes<CustomLayers...> > network;
 
-  //! The matrix of responses to the input data points.  This member is empty,
-  //! except during training.
-  MatType responses;
+  //! The matrix of data points (predictors).
+  arma::mat predictors;
 
-  //! Locally-stored output of the network from a forward pass; used by the
-  //! backward pass.
-  MatType networkOutput;
-  //! Locally-stored output of the backward pass; used by the gradient pass.
-  MatType networkDelta;
-  //! Locally-stored error of the backward pass; used by the gradient pass.
-  MatType error;
+  //! The matrix of responses to the input data points.
+  arma::mat responses;
 
-  //! If true, each layer has its memory properly set for a forward/backward
-  //! pass.
-  bool layerMemoryIsSet;
+  //! Matrix of (trained) parameters.
+  arma::mat parameter;
 
-  //! If true, each layer has its inputDimensions properly set, and
-  //! `totalInputSize` and `totalOutputSize` are valid.
-  bool inputDimensionsAreSet;
+  //! The number of separable functions (the number of predictor points).
+  size_t numFunctions;
 
-  // RNN will call `CheckNetwork()`, which is private.
-  friend class RNN<OutputLayerType, InitializationRuleType, MatType>;
+  //! The current error for the backward pass.
+  arma::mat error;
+
+  //! Locally-stored delta visitor.
+  DeltaVisitor deltaVisitor;
+
+  //! Locally-stored output parameter visitor.
+  OutputParameterVisitor outputParameterVisitor;
+
+  //! Locally-stored weight size visitor.
+  WeightSizeVisitor weightSizeVisitor;
+
+  //! Locally-stored output width visitor.
+  OutputWidthVisitor outputWidthVisitor;
+
+  //! Locally-stored output height visitor.
+  OutputHeightVisitor outputHeightVisitor;
+
+  //! Locally-stored loss visitor
+  LossVisitor lossVisitor;
+
+  //! Locally-stored reset visitor.
+  ResetVisitor resetVisitor;
+
+  //! Locally-stored delete visitor.
+  DeleteVisitor deleteVisitor;
+
+  //! The current evaluation mode (training or testing).
+  bool deterministic;
+
+  //! Locally-stored delta object.
+  arma::mat delta;
+
+  //! Locally-stored input parameter object.
+  arma::mat inputParameter;
+
+  //! Locally-stored output parameter object.
+  arma::mat outputParameter;
+
+  //! Locally-stored gradient parameter.
+  arma::mat gradient;
+
+  //! Locally-stored copy visitor
+  CopyVisitor<CustomLayers...> copyVisitor;
+
+  // The GAN class should have access to internal members.
+  template<
+    typename Model,
+    typename InitializerType,
+    typename NoiseType,
+    typename PolicyType
+  >
+  friend class GAN;
 }; // class FFN
 
 } // namespace ann

src/mlpack/methods/ann/forward_decls.hpp

@@ -1,29 +0,0 @@
-/**
- * @file forward_decls.hpp
- * @author Ryan Curtin
- *
- * Forward declarations of network types.  This is needed for some `friend`
- * functionality.
- */
-#ifndef MLPACK_METHODS_ANN_FORWARD_DECLS_HPP
-#define MLPACK_METHODS_ANN_FORWARD_DECLS_HPP
-
-namespace mlpack {
-namespace ann {
-
-// See ffn.hpp.
-template<typename OutputLayerType,
-         typename InitializationRuleType,
-         typename MatType>
-class FFN;
-
-// See rnn.hpp.
-template<typename OutputLayerType,
-         typename InitializationRuleType,
-         typename MatType>
-class RNN;
-
-} // namespace ann
-} // namespace mlpack
-
-#endif

src/mlpack/methods/ann/init_rules/const_init.hpp

@@ -98,13 +98,7 @@ class ConstInitialization
   //! Get the initialization value.
   double const& InitValue() const { return initVal; }
   //! Modify the initialization value.
-  double& InitValue() { return initVal; }
-
-  template<typename Archive>
-  void serialize(Archive& ar, const uint32_t /* version */)
-  {
-    ar(CEREAL_NVP(initVal));
-  }
+  double& initValue() { return initVal; }
 
  private:
   //! Value to be initialized with

src/mlpack/methods/ann/init_rules/glorot_init.hpp

@@ -104,19 +104,13 @@ class GlorotInitializationType
    */
   template<typename eT>
   void Initialize(arma::Cube<eT>& W);
-
-  /**
-   * Serialize the initialization.  (Nothing to serialize for this one.)
-   */
-  template<typename Archive>
-  void serialize(Archive& /* ar */, const uint32_t /* version */) { }
 }; // class GlorotInitializationType
 
-template<>
+template <>
 template<typename eT>
 inline void GlorotInitializationType<false>::Initialize(arma::Mat<eT>& W,
-                                                        const size_t rows,
-                                                        const size_t cols)
+                                                       const size_t rows,
+                                                       const size_t cols)
 {
   if (W.is_empty())
     W.set_size(rows, cols);
@@ -126,7 +120,7 @@ inline void GlorotInitializationType<false>::Initialize(arma::Mat<eT>& W,
   normalInit.Initialize(W, rows, cols);
 }
 
-template<>
+template <>
 template<typename eT>
 inline void GlorotInitializationType<false>::Initialize(arma::Mat<eT>& W)
 {
@@ -138,7 +132,7 @@ inline void GlorotInitializationType<false>::Initialize(arma::Mat<eT>& W)
   normalInit.Initialize(W);
 }
 
-template<>
+template <>
 template<typename eT>
 inline void GlorotInitializationType<true>::Initialize(arma::Mat<eT>& W,
                                                        const size_t rows,
@@ -153,7 +147,7 @@ inline void GlorotInitializationType<true>::Initialize(arma::Mat<eT>& W,
   randomInit.Initialize(W, rows, cols);
 }
 
-template<>
+template <>
 template<typename eT>
 inline void GlorotInitializationType<true>::Initialize(arma::Mat<eT>& W)
 {

src/mlpack/methods/ann/init_rules/he_init.hpp

@@ -136,12 +136,6 @@ class HeInitialization
     for (size_t i = 0; i < W.n_slices; ++i)
       Initialize(W.slice(i));
   }
-
-  template<typename Archive>
-  void serialize(Archive& /* ar */, const uint32_t /* version */)
-  {
-    // Nothing to do.
-  }
 }; // class HeInitialization
 
 } // namespace ann

src/mlpack/methods/ann/init_rules/network_init.hpp

@@ -14,10 +14,14 @@
 #define MLPACK_METHODS_ANN_INIT_RULES_NETWORK_INIT_HPP
 
 #include <mlpack/prereqs.hpp>
-#include <mlpack/methods/ann/layer/layer.hpp>
 
+#include "../visitor/reset_visitor.hpp"
+#include "../visitor/weight_size_visitor.hpp"
+#include "../visitor/weight_set_visitor.hpp"
 #include "init_rules_traits.hpp"
 
+#include <mlpack/methods/ann/layer/layer_types.hpp>
+
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
@@ -25,7 +29,7 @@ namespace ann /** Artificial Neural Network. */ {
  * This class is used to initialize the network with the given initialization
  * rule.
  */
-template<typename InitializationRuleType>
+template<typename InitializationRuleType, typename... CustomLayers>
 class NetworkInitialization
 {
  public:
@@ -50,18 +54,16 @@ class NetworkInitialization
    * @param parameterOffset Offset for network paramater, default 0.
    */
   template <typename eT>
-  void Initialize(const std::vector<Layer<arma::Mat<eT>>*>& network,
-                  arma::Mat<eT>& parameters,
-                  size_t parameterOffset = 0)
+  void Initialize(const std::vector<LayerTypes<CustomLayers...> >& network,
+                  arma::Mat<eT>& parameter, size_t parameterOffset = 0)
   {
-    // Determine the total number of parameters/weights of the given network.
-    if (parameters.is_empty())
+    // Determine the number of parameter/weights of the given network.
+    if (parameter.is_empty())
     {
       size_t weights = 0;
       for (size_t i = 0; i < network.size(); ++i)
-        weights += network[i]->WeightSize();
-
-      parameters.set_size(weights, 1);
+        weights += boost::apply_visitor(weightSizeVisitor, network[i]);
+      parameter.set_size(weights, 1);
     }
 
     // Initialize the network layer by layer or the complete network.
@@ -71,8 +73,9 @@ class NetworkInitialization
       {
         // Initialize the layer with the specified parameter/weight
         // initialization rule.
-        const size_t weight = network[i]->WeightSize();
-        arma::Mat<eT> tmp = arma::mat(parameters.memptr() + offset,
+        const size_t weight = boost::apply_visitor(weightSizeVisitor,
+            network[i]);
+        arma::Mat<eT> tmp = arma::mat(parameter.memptr() + offset,
             weight, 1, false, false);
         initializeRule.Initialize(tmp, tmp.n_elem, 1);
 
@@ -82,7 +85,19 @@ class NetworkInitialization
     }
     else
     {
-      initializeRule.Initialize(parameters, parameters.n_elem, 1);
+      initializeRule.Initialize(parameter, parameter.n_elem, 1);
+    }
+
+    // Note: We can't merge the for loop into the for loop above because
+    // WeightSetVisitor also sets the parameter/weights of the inner modules.
+    // Inner Modules are held by the parent module e.g. the concat module can
+    // hold various other modules.
+    for (size_t i = 0, offset = parameterOffset; i < network.size(); ++i)
+    {
+      offset += boost::apply_visitor(WeightSetVisitor(parameter, offset),
+          network[i]);
+
+      boost::apply_visitor(resetVisitor, network[i]);
     }
   }
 
@@ -90,6 +105,12 @@ class NetworkInitialization
   //! Instantiated InitializationRule object for initializing the network
   //! parameter.
   InitializationRuleType initializeRule;
+
+  //! Locally-stored reset visitor.
+  ResetVisitor resetVisitor;
+
+  //! Locally-stored weight size visitor.
+  WeightSizeVisitor weightSizeVisitor;
 }; // class NetworkInitialization
 
 } // namespace ann

src/mlpack/methods/ann/init_rules/oivs_init.hpp

@@ -47,13 +47,15 @@ namespace ann /** Artificial Neural Network. */ {
  * w_i &=& \hat{w} \cdot \sqrt{a_i + 1}
  * @f}
  *
- * Where f is the transfer function epsilon, k custom parameters, n the number
- * of neurons in the outgoing layer and gamma a parameter that defines the
- * random interval.
+ * Where f is the transfer function epsilon, k custom parameters, n the number of
+ * neurons in the outgoing layer and gamma a parameter that defines the random
+ * interval.
  *
  * @tparam ActivationFunction The activation function used for the oivs method.
  */
-template<class ActivationFunction = LogisticFunction>
+template<
+    class ActivationFunction = LogisticFunction
+>
 class OivsInitialization
 {
  public:

src/mlpack/methods/ann/init_rules/random_init.hpp

@@ -115,13 +115,6 @@ class RandomInitialization
       Initialize(W.slice(i));
   }
 
-  template<typename Archive>
-  void serialize(Archive& ar, const uint32_t /* version */)
-  {
-    ar(CEREAL_NVP(lowerBound));
-    ar(CEREAL_NVP(upperBound));
-  }
-
  private:
   //! The number used as lower bound.
   double lowerBound;

src/mlpack/methods/ann/layer/CMakeLists.txt

@@ -3,18 +3,67 @@
 set(SOURCES
   add.hpp
   add_impl.hpp
+  add_merge.hpp
+  add_merge_impl.hpp
+  adaptive_max_pooling.hpp
+  adaptive_max_pooling_impl.hpp
+  adaptive_mean_pooling.hpp
+  adaptive_mean_pooling_impl.hpp
   alpha_dropout.hpp
   alpha_dropout_impl.hpp
+  atrous_convolution.hpp
+  atrous_convolution_impl.hpp
   base_layer.hpp
+  batch_norm.hpp
+  batch_norm_impl.hpp
+  bicubic_interpolation.hpp
+  bicubic_interpolation_impl.hpp
+  bilinear_interpolation.hpp
+  bilinear_interpolation_impl.hpp
+  channel_shuffle.hpp
+  channel_shuffle_impl.hpp
+  concat.hpp
+  concat_impl.hpp
+  concat_performance.hpp
+  concat_performance_impl.hpp
   concatenate.hpp
   concatenate_impl.hpp
+  constant.hpp
+  constant_impl.hpp
   convolution.hpp
   convolution_impl.hpp
   dropconnect.hpp
   dropconnect_impl.hpp
   dropout.hpp
   dropout_impl.hpp
+  elu.hpp
+  elu_impl.hpp
+  fast_lstm.hpp
+  fast_lstm_impl.hpp
+  flatten_t_swish.hpp
+  flatten_t_swish_impl.hpp
+  flexible_relu.hpp
+  flexible_relu_impl.hpp
+  glimpse.hpp
+  glimpse_impl.hpp
+  group_norm.hpp
+  group_norm_impl.hpp
+  gru.hpp
+  gru_impl.hpp
+  hard_tanh.hpp
+  hard_tanh_impl.hpp
+  highway.hpp
+  highway_impl.hpp
+  instance_norm.hpp
+  instance_norm_impl.hpp
+  isrlu.hpp
+  isrlu_impl.hpp
+  join.hpp
+  join_impl.hpp
   layer.hpp
+  layer_norm.hpp
+  layer_norm_impl.hpp
+  layer_traits.hpp
   layer_types.hpp
   leaky_relu.hpp
   leaky_relu_impl.hpp
@@ -22,22 +71,73 @@ set(SOURCES
   linear_impl.hpp
   linear_no_bias.hpp
   linear_no_bias_impl.hpp
-  linear3d.hpp
-  linear3d_impl.hpp
   log_softmax.hpp
   log_softmax_impl.hpp
+  lookup.hpp
+  lookup_impl.hpp
+  lp_pooling.hpp
+  lp_pooling_impl.hpp
   lstm.hpp
   lstm_impl.hpp
   max_pooling.hpp
   max_pooling_impl.hpp
-  multi_layer.hpp
-  multi_layer_impl.hpp
+  mean_pooling.hpp
+  mean_pooling_impl.hpp
+  minibatch_discrimination.hpp
+  minibatch_discrimination_impl.hpp
+  multihead_attention_impl.hpp
+  multihead_attention.hpp
+  multiply_constant.hpp
+  multiply_constant_impl.hpp
+  multiply_merge.hpp
+  multiply_merge_impl.hpp
+  nearest_interpolation.hpp
+  nearest_interpolation_impl.hpp
   noisylinear.hpp
   noisylinear_impl.hpp
-  padding.hpp
+  parametric_relu.hpp
+  parametric_relu_impl.hpp
+  pixel_shuffle.hpp
+  pixel_shuffle_impl.hpp
+  positional_encoding.hpp
+  positional_encoding_impl.hpp
+  recurrent.hpp
+  recurrent_impl.hpp
+  recurrent_attention.hpp
+  recurrent_attention_impl.hpp
+  reinforce_normal.hpp
+  reinforce_normal_impl.hpp
+  relu6.hpp
+  relu6_impl.hpp
+  reparametrization.hpp
+  reparametrization_impl.hpp
   radial_basis_function.hpp
   radial_basis_function_impl.hpp
-  serialization.hpp
+  select.hpp
+  select_impl.hpp
+  sequential.hpp
+  sequential_impl.hpp
+  softmax_impl.hpp
+  softmax.hpp
+  spatial_dropout.hpp
+  spatial_dropout_impl.hpp
+  subview.hpp
+  transposed_convolution.hpp
+  transposed_convolution_impl.hpp
+  vr_class_reward.hpp
+  vr_class_reward_impl.hpp
+  c_relu.hpp
+  c_relu_impl.hpp
+  weight_norm.hpp
+  weight_norm_impl.hpp
+  hardshrink.hpp
+  hardshrink_impl.hpp
+  celu.hpp
+  celu_impl.hpp
+  softshrink.hpp
+  softshrink_impl.hpp
+  softmin.hpp
+  softmin_impl.hpp
 )
 
 # Add directory name to sources.

src/mlpack/methods/ann/layer/adaptive_max_pooling.hpp

@@ -13,9 +13,7 @@
 #define MLPACK_METHODS_ANN_LAYER_ADAPTIVE_MAX_POOLING_HPP
 
 #include <mlpack/prereqs.hpp>
-
-#include "layer.hpp"
-#include "max_pooling.hpp"
+#include "layer_types.hpp"
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
@@ -23,17 +21,20 @@ namespace ann /** Artificial Neural Network. */ {
 /**
  * Implementation of the AdaptiveMaxPooling layer.
  *
- * @tparam InputType The type of the layer's inputs. The layer automatically
- *    cast inputs to this type (Default: arma::mat).
- * @tparam OutputType The type of the layer's Outputs. The layer automatically
- *    cast inputs to this type (Default: arma::mat).
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
  */
-template <typename InputType = arma::mat, typename OutputType = arma::mat>
-class AdaptiveMaxPoolingType : public Layer<InputType, OutputType>
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+class AdaptiveMaxPooling
 {
  public:
   //! Create the AdaptiveMaxPooling object.
-  AdaptiveMaxPoolingType();
+  AdaptiveMaxPooling();
 
   /**
    * Create the AdaptiveMaxPooling object.
@@ -41,16 +42,15 @@ class AdaptiveMaxPoolingType : public Layer<InputType, OutputType>
    * @param outputWidth Width of the output.
    * @param outputHeight Height of the output.
    */
-  AdaptiveMaxPoolingType(const size_t outputWidth,
-                         const size_t outputHeight);
+  AdaptiveMaxPooling(const size_t outputWidth,
+                     const size_t outputHeight);
 
   /**
    * Create the AdaptiveMaxPooling object.
    *
-   * @param outputShape A two-value tuple indicating width and height of the
-   *     output.
+   * @param outputShape A two-value tuple indicating width and height of the output.
    */
-  AdaptiveMaxPoolingType(const std::tuple<size_t, size_t>& outputShape);
+  AdaptiveMaxPooling(const std::tuple<size_t, size_t>& outputShape);
 
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function
@@ -59,7 +59,8 @@ class AdaptiveMaxPoolingType : public Layer<InputType, OutputType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
-  void Forward(const InputType& input, OutputType& output);
+  template<typename eT>
+  void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);
 
   /**
    * Ordinary feed backward pass of a neural network, using 3rd-order tensors as
@@ -70,30 +71,48 @@ class AdaptiveMaxPoolingType : public Layer<InputType, OutputType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const InputType& input,
-                const OutputType& gy,
-                OutputType& g);
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& input,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g);
+
+  //! Get the output parameter.
+  const OutputDataType& OutputParameter() const
+  { return poolingLayer.OutputParameter(); }
+
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return poolingLayer.OutputParameter(); }
+
+  //! Get the delta.
+  const OutputDataType& Delta() const { return poolingLayer.Delta(); }
+  //! Modify the delta.
+  OutputDataType& Delta() { return poolingLayer.Delta(); }
+
+  //! Get the input width.
+  size_t InputWidth() const { return poolingLayer.InputWidth(); }
+  //! Modify the input width.
+  size_t& InputWidth() { return poolingLayer.InputWidth(); }
+
+  //! Get the input height.
+  size_t InputHeight() const { return poolingLayer.InputHeight(); }
+  //! Modify the input height.
+  size_t& InputHeight() { return poolingLayer.InputHeight(); }
 
   //! Get the output width.
-  size_t const& OutputWidth() const { return outputWidth; }
+  size_t OutputWidth() const { return outputWidth; }
   //! Modify the output width.
   size_t& OutputWidth() { return outputWidth; }
 
   //! Get the output height.
-  size_t const& OutputHeight() const { return outputHeight; }
+  size_t OutputHeight() const { return outputHeight; }
   //! Modify the output height.
   size_t& OutputHeight() { return outputHeight; }
 
-  //! Get the number of trainable weights.
-  size_t WeightSize() const { return 0; }
+  //! Get the input size.
+  size_t InputSize() const { return poolingLayer.InputSize(); }
 
-  const std::vector<size_t>& OutputDimensions() const
-  {
-    std::vector<size_t> result(this->inputDimensions.size(), 1);
-    result[0] = outputWidth;
-    result[1] = outputHeight;
-    return result;
-  }
+  //! Get the output size.
+  size_t OutputSize() const { return poolingLayer.OutputSize(); }
 
   //! Get the size of the weights.
   size_t WeightSize() const { return 0; }
@@ -108,7 +127,7 @@ class AdaptiveMaxPoolingType : public Layer<InputType, OutputType>
   /**
    * Initialize Kernel Size and Stride for Adaptive Pooling.
    */
-  void InitializeAdaptivePadding()
+  void IntializeAdaptivePadding()
   {
     poolingLayer.StrideWidth() = std::floor(poolingLayer.InputWidth() /
         outputWidth);
@@ -131,7 +150,7 @@ class AdaptiveMaxPoolingType : public Layer<InputType, OutputType>
   }
 
   //! Locally stored MaxPooling Object.
-  MaxPoolingType<InputType, OutputType> poolingLayer;
+  MaxPooling<InputDataType, OutputDataType> poolingLayer;
 
   //! Locally-stored output width.
   size_t outputWidth;
@@ -141,12 +160,7 @@ class AdaptiveMaxPoolingType : public Layer<InputType, OutputType>
 
   //! Locally-stored reset parameter used to initialize the layer once.
   bool reset;
-}; // class AdaptiveMaxPoolingType
-
-// Convenience typedefs.
-
-// Standard Adaptive max pooling layer.
-typedef AdaptiveMaxPoolingType<arma::mat, arma::mat> AdaptiveMaxPooling;
+}; // class AdaptiveMaxPooling
 
 } // namespace ann
 } // namespace mlpack

src/mlpack/methods/ann/layer/adaptive_max_pooling_impl.hpp

@@ -18,61 +18,61 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template<typename InputType, typename OutputType>
-AdaptiveMaxPoolingType<InputType, OutputType>::AdaptiveMaxPoolingType()
+template<typename InputDataType, typename OutputDataType>
+AdaptiveMaxPooling<InputDataType, OutputDataType>::AdaptiveMaxPooling()
 {
   // Nothing to do here.
 }
 
-template <typename InputType, typename OutputType>
-AdaptiveMaxPoolingType<InputType, OutputType>::AdaptiveMaxPoolingType(
+template <typename InputDataType, typename OutputDataType>
+AdaptiveMaxPooling<InputDataType, OutputDataType>::AdaptiveMaxPooling(
     const size_t outputWidth,
     const size_t outputHeight) :
-    AdaptiveMaxPoolingType(std::tuple<size_t, size_t>(outputWidth, outputHeight))
+    AdaptiveMaxPooling(std::tuple<size_t, size_t>(outputWidth, outputHeight))
 {
   // Nothing to do here.
 }
 
-template <typename InputType, typename OutputType>
-AdaptiveMaxPoolingType<InputType, OutputType>::AdaptiveMaxPoolingType(
+template <typename InputDataType, typename OutputDataType>
+AdaptiveMaxPooling<InputDataType, OutputDataType>::AdaptiveMaxPooling(
     const std::tuple<size_t, size_t>& outputShape):
     outputWidth(std::get<0>(outputShape)),
     outputHeight(std::get<1>(outputShape)),
     reset(false)
 {
-  poolingLayer = ann::MaxPoolingType<InputType, OutputType>(0, 0);
+  poolingLayer = ann::MaxPooling<>(0, 0);
 }
 
-template<typename InputType, typename OutputType>
-void AdaptiveMaxPoolingType<InputType, OutputType>::Forward(
-    const InputType& input, OutputType& output)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void AdaptiveMaxPooling<InputDataType, OutputDataType>::Forward(
+    const arma::Mat<eT>& input, arma::Mat<eT>& output)
 {
   if (!reset)
   {
-    InitializeAdaptivePadding();
+    IntializeAdaptivePadding();
     reset = true;
   }
 
   poolingLayer.Forward(input, output);
 }
 
-template<typename InputType, typename OutputType>
-void AdaptiveMaxPoolingType<InputType, OutputType>::Backward(
-  const InputType& input,
-  const OutputType& gy,
-  OutputType& g)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void AdaptiveMaxPooling<InputDataType, OutputDataType>::Backward(
+  const arma::Mat<eT>& input,
+  const arma::Mat<eT>& gy,
+  arma::Mat<eT>& g)
 {
   poolingLayer.Backward(input, gy, g);
 }
 
-template<typename InputType, typename OutputType>
+template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
-void AdaptiveMaxPoolingType<InputType, OutputType>::serialize(
+void AdaptiveMaxPooling<InputDataType, OutputDataType>::serialize(
     Archive& ar,
     const uint32_t /* version */)
 {
-  ar(cereal::base_class<Layer<InputType, OutputType>>(this));
-
   ar(CEREAL_NVP(outputWidth));
   ar(CEREAL_NVP(outputHeight));
   ar(CEREAL_NVP(reset));

src/mlpack/methods/ann/layer/adaptive_mean_pooling.hpp

@@ -1,4 +1,3 @@
-// Maybe 
 /**
  * @file methods/ann/layer/adaptive_mean_pooling.hpp
  * @author Kartik Dutt
@@ -15,9 +14,7 @@
 #define MLPACK_METHODS_ANN_LAYER_ADAPTIVE_MEAN_POOLING_HPP
 
 #include <mlpack/prereqs.hpp>
-
-#include "layer.hpp"
-#include "mean_pooling.hpp"
+#include "layer_types.hpp"
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
@@ -25,17 +22,20 @@ namespace ann /** Artificial Neural Network. */ {
 /**
  * Implementation of the AdaptiveMeanPooling.
  *
- * @tparam InputType The type of the layer's inputs. The layer automatically
- *    cast inputs to this type (Default: arma::mat).
- * @tparam OutputType The type of the layer's Outputs. The layer automatically
- *    cast inputs to this type (Default: arma::mat).
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
  */
-template <typename InputType = arma::mat, typename OutputType = arma::mat>
-class AdaptiveMeanPoolingType : public Layer<InputType, OutputType>
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+class AdaptiveMeanPooling
 {
  public:
   //! Create the AdaptiveMeanPooling object.
-  AdaptiveMeanPoolingType();
+  AdaptiveMeanPooling();
 
   /**
    * Create the AdaptiveMeanPooling object.
@@ -43,16 +43,15 @@ class AdaptiveMeanPoolingType : public Layer<InputType, OutputType>
    * @param outputWidth Width of the output.
    * @param outputHeight Height of the output.
    */
-  AdaptiveMeanPoolingType(const size_t outputWidth,
-                          const size_t outputHeight);
+  AdaptiveMeanPooling(const size_t outputWidth,
+                      const size_t outputHeight);
 
   /**
    * Create the AdaptiveMeanPooling object.
    *
-   * @param outputShape A two-value tuple indicating width and height of the
-   *      output.
+   * @param outputShape A two-value tuple indicating width and height of the output.
    */
-  AdaptiveMeanPoolingType(const std::tuple<size_t, size_t>& outputShape);
+  AdaptiveMeanPooling(const std::tuple<size_t, size_t>& outputShape);
 
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function
@@ -61,7 +60,8 @@ class AdaptiveMeanPoolingType : public Layer<InputType, OutputType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
-  void Forward(const InputType& input, OutputType& output);
+  template<typename eT>
+  void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);
 
   /**
    * Ordinary feed backward pass of a neural network, using 3rd-order tensors as
@@ -72,30 +72,48 @@ class AdaptiveMeanPoolingType : public Layer<InputType, OutputType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const InputType& input,
-                const OutputType& gy,
-                OutputType& g);
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& input,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g);
+
+  //! Get the output parameter.
+  const OutputDataType& OutputParameter() const
+  { return poolingLayer.OutputParameter(); }
+
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return poolingLayer.OutputParameter(); }
+
+  //! Get the delta.
+  const OutputDataType& Delta() const { return poolingLayer.Delta(); }
+  //! Modify the delta.
+  OutputDataType& Delta() { return poolingLayer.Delta(); }
+
+  //! Get the input width.
+  size_t InputWidth() const { return poolingLayer.InputWidth(); }
+  //! Modify the input width.
+  size_t& InputWidth() { return poolingLayer.InputWidth(); }
+
+  //! Get the input height.
+  size_t InputHeight() const { return poolingLayer.InputHeight(); }
+  //! Modify the input height.
+  size_t& InputHeight() { return poolingLayer.InputHeight(); }
 
   //! Get the output width.
-  size_t const& OutputWidth() const { return outputWidth; }
+  size_t OutputWidth() const { return outputWidth; }
   //! Modify the output width.
   size_t& OutputWidth() { return outputWidth; }
 
   //! Get the output height.
-  size_t const& OutputHeight() const { return outputHeight; }
+  size_t OutputHeight() const { return outputHeight; }
   //! Modify the output height.
   size_t& OutputHeight() { return outputHeight; }
 
-  //! Get the number of trainable weights.
-  size_t WeightSize() const { return 0; }
+  //! Get the input size.
+  size_t InputSize() const { return poolingLayer.InputSize(); }
 
-  const std::vector<size_t>& OutputDimensions() const
-  {
-    std::vector<size_t> result(this->inputDimensions);
-    result[0] = outputWidth;
-    result[1] = outputHeight;
-    return result;
-  }
+  //! Get the output size.
+  size_t OutputSize() const { return poolingLayer.OutputSize(); }
 
   //! Get the size of the weights.
   size_t WeightSize() const { return 0; }
@@ -110,7 +128,7 @@ class AdaptiveMeanPoolingType : public Layer<InputType, OutputType>
   /**
    * Initialize Kernel Size and Stride for Adaptive Pooling.
    */
-  void InitializeAdaptivePadding()
+  void IntializeAdaptivePadding()
   {
     poolingLayer.StrideWidth() = std::floor(poolingLayer.InputWidth() /
         outputWidth);
@@ -133,7 +151,7 @@ class AdaptiveMeanPoolingType : public Layer<InputType, OutputType>
   }
 
   //! Locally stored MeanPooling Object.
-  MeanPoolingType<InputType, OutputType> poolingLayer;
+  MeanPooling<InputDataType, OutputDataType> poolingLayer;
 
   //! Locally-stored output width.
   size_t outputWidth;
@@ -143,12 +161,7 @@ class AdaptiveMeanPoolingType : public Layer<InputType, OutputType>
 
   //! Locally-stored reset parameter used to initialize the layer once.
   bool reset;
-}; // class AdaptiveMeanPoolingType
-
-// Convenience typedefs.
-
-// Standard Adaptive mean pooling layer.
-typedef AdaptiveMeanPoolingType<arma::mat, arma::mat> AdaptiveMeanPooling;
+}; // class AdaptiveMeanPooling
 
 } // namespace ann
 } // namespace mlpack

src/mlpack/methods/ann/layer/adaptive_mean_pooling_impl.hpp

@@ -18,61 +18,61 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template<typename InputType, typename OutputType>
-AdaptiveMeanPoolingType<InputType, OutputType>::AdaptiveMeanPoolingType()
+template<typename InputDataType, typename OutputDataType>
+AdaptiveMeanPooling<InputDataType, OutputDataType>::AdaptiveMeanPooling()
 {
   // Nothing to do here.
 }
 
-template <typename InputType, typename OutputType>
-AdaptiveMeanPoolingType<InputType, OutputType>::AdaptiveMeanPoolingType(
+template <typename InputDataType, typename OutputDataType>
+AdaptiveMeanPooling<InputDataType, OutputDataType>::AdaptiveMeanPooling(
     const size_t outputWidth,
     const size_t outputHeight) :
-    AdaptiveMeanPoolingType(std::tuple<size_t, size_t>(outputWidth, outputHeight))
+    AdaptiveMeanPooling(std::tuple<size_t, size_t>(outputWidth, outputHeight))
 {
   // Nothing to do here.
 }
 
-template <typename InputType, typename OutputType>
-AdaptiveMeanPoolingType<InputType, OutputType>::AdaptiveMeanPoolingType(
+template <typename InputDataType, typename OutputDataType>
+AdaptiveMeanPooling<InputDataType, OutputDataType>::AdaptiveMeanPooling(
     const std::tuple<size_t, size_t>& outputShape):
     outputWidth(std::get<0>(outputShape)),
     outputHeight(std::get<1>(outputShape)),
     reset(false)
 {
-  poolingLayer = ann::MeanPoolingType<InputType, OutputType>(0, 0);
+  poolingLayer = ann::MeanPooling<>(0, 0);
 }
 
-template<typename InputType, typename OutputType>
-void AdaptiveMeanPoolingType<InputType, OutputType>::Forward(
-    const InputType& input, OutputType& output)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void AdaptiveMeanPooling<InputDataType, OutputDataType>::Forward(
+    const arma::Mat<eT>& input, arma::Mat<eT>& output)
 {
   if (!reset)
   {
-    InitializeAdaptivePadding();
+    IntializeAdaptivePadding();
     reset = true;
   }
 
   poolingLayer.Forward(input, output);
 }
 
-template<typename InputType, typename OutputType>
-void AdaptiveMeanPoolingType<InputType, OutputType>::Backward(
-  const InputType& input,
-  const OutputType& gy,
-  OutputType& g)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void AdaptiveMeanPooling<InputDataType, OutputDataType>::Backward(
+  const arma::Mat<eT>& input,
+  const arma::Mat<eT>& gy,
+  arma::Mat<eT>& g)
 {
   poolingLayer.Backward(input, gy, g);
 }
 
-template<typename InputType, typename OutputType>
+template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
-void AdaptiveMeanPoolingType<InputType, OutputType>::serialize(
+void AdaptiveMeanPooling<InputDataType, OutputDataType>::serialize(
     Archive& ar,
     const uint32_t /* version */)
 {
-  ar(cereal::base_class<Layer<InputType, OutputType>>(this));
-
   ar(CEREAL_NVP(outputWidth));
   ar(CEREAL_NVP(outputHeight));
   ar(CEREAL_NVP(reset));

src/mlpack/methods/ann/layer/add.hpp

@@ -13,90 +13,98 @@
 #define MLPACK_METHODS_ANN_LAYER_ADD_HPP
 
 #include <mlpack/prereqs.hpp>
-#include "layer.hpp"
+#include <mlpack/methods/ann/layer/layer_traits.hpp>
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
 /**
- * Implementation of the Add layer. The Add module applies a bias term to the
- * incoming data.
+ * Implementation of the Add module class. The Add module applies a bias term
+ * to the incoming data.
  *
- * @tparam MatType Matrix representation to accept as input and use for
- *    computation.
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
  */
-template<typename MatType>
-class AddType : public Layer<MatType>
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+class Add
 {
  public:
   /**
-   * Create the AddType object.  The output size of the layer will be the same
-   * as the input size.
+   * Create the Add object using the specified number of output units.
+   *
+   * @param outSize The number of output units.
    */
-  AddType();
-
-  //! Clone the AddType object. This handles polymorphism correctly.
-  AddType* Clone() const { return new AddType(*this); }
-
-  // Virtual destructor.
-  virtual ~AddType() { }
-
-  //! Copy the given AddType layer.
-  AddType(const AddType& other);
-  //! Take ownership of the given AddType layer.
-  AddType(AddType&& other);
-  //! Copy the given AddType layer.
-  AddType& operator=(const AddType& other);
-  //! Take ownership of the given AddType layer.
-  AddType& operator=(AddType&& other);
+  Add(const size_t outSize = 0);
 
   /**
-   * Forward pass: add the bias to the input.
+   * Ordinary feed forward pass of a neural network, evaluating the function
+   * f(x) by propagating the activity forward through f.
    *
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
-  void Forward(const MatType& input, MatType& output);
+  template<typename eT>
+  void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);
 
   /**
-   * Backward pass: send weights backwards (the bias does not affect anything).
+   * Ordinary feed backward pass of a neural network, calculating the function
+   * f(x) by propagating x backwards trough f. Using the results from the feed
+   * forward pass.
    *
    * @param * (input) The propagated input activation.
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const MatType& /* input */,
-                const MatType& gy,
-                MatType& g);
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& /* input */,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g);
 
   /**
-   * Calculate the gradient using the output and the input activation.
+   * Calculate the gradient using the output delta and the input activation.
    *
    * @param * (input) The propagated input.
    * @param error The calculated error.
    * @param gradient The calculated gradient.
    */
-  void Gradient(const MatType& /* input */,
-                const MatType& error,
-                MatType& gradient);
+  template<typename eT>
+  void Gradient(const arma::Mat<eT>& /* input */,
+                const arma::Mat<eT>& error,
+                arma::Mat<eT>& gradient);
 
-  //! Return the weights of the network.
-  const MatType& Parameters() const { return weights; }
-  //! Modify the weights of the network.
-  MatType& Parameters() { return weights; }
+  //! Get the parameters.
+  OutputDataType const& Parameters() const { return weights; }
+  //! Modify the parameters.
+  OutputDataType& Parameters() { return weights; }
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  //! Get the gradient.
+  OutputDataType const& Gradient() const { return gradient; }
+  //! Modify the gradient.
+  OutputDataType& Gradient() { return gradient; }
+
+  //! Get the output size.
+  size_t OutputSize() const { return outSize; }
 
   //! Get the size of weights.
   size_t WeightSize() const { return outSize; }
 
-  //! Compute the output dimensions of the layer, based on the internal values
-  //! of `InputDimensions()`.
-  void ComputeOutputDimensions();
-
-  //! Set the weights of the layer to use the given memory.
-  void SetWeights(typename MatType::elem_type* weightPtr);
-
   /**
-   * Serialize the layer.
+   * Serialize the layer
    */
   template<typename Archive>
   void serialize(Archive& ar, const uint32_t /* version */);
@@ -106,11 +114,17 @@ class AddType : public Layer<MatType>
   size_t outSize;
 
   //! Locally-stored weight object.
-  MatType weights;
-}; // class Add
+  OutputDataType weights;
 
-// Standard Add layer.
-typedef AddType<arma::mat> Add;
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored gradient object.
+  OutputDataType gradient;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+}; // class Add
 
 } // namespace ann
 } // namespace mlpack

src/mlpack/methods/ann/layer/add_impl.hpp

@@ -19,103 +19,51 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template<typename MatType>
-AddType<MatType>::AddType() : outSize(0)
+template<typename InputDataType, typename OutputDataType>
+Add<InputDataType, OutputDataType>::Add(const size_t outSize) :
+    outSize(outSize)
 {
-  // Nothing to do.
+  weights.set_size(WeightSize(), 1);
 }
 
-template<typename MatType>
-AddType<MatType>::AddType(const AddType& other) :
-    Layer<MatType>(other),
-    outSize(other.outSize)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void Add<InputDataType, OutputDataType>::Forward(
+    const arma::Mat<eT>& input, arma::Mat<eT>& output)
 {
-  // Nothing to do.
+  output = input;
+  output.each_col() += weights;
 }
 
-template<typename MatType>
-AddType<MatType>::AddType(AddType&& other) :
-    Layer<MatType>(std::move(other)),
-    outSize(std::move(other.outSize))
-{
-  // Nothing to do.
-}
-
-template<typename MatType>
-AddType<MatType>&
-AddType<MatType>::operator=(const AddType& other)
-{
-  if (&other != this)
-  {
-    Layer<MatType>::operator=(other);
-    outSize = other.outSize;
-  }
-
-  return *this;
-}
-
-template<typename MatType>
-AddType<MatType>&
-AddType<MatType>::operator=(AddType&& other)
-{
-  if (&other != this)
-  {
-    Layer<MatType>::operator=(std::move(other));
-    outSize = std::move(other.outSize);
-  }
-
-  return *this;
-}
-
-template<typename MatType>
-void AddType<MatType>::Forward(const MatType& input, MatType& output)
-{
-  output = input + arma::repmat(arma::vectorise(weights), 1, input.n_cols);
-}
-
-template<typename MatType>
-void AddType<MatType>::Backward(
-    const MatType& /* input */,
-    const MatType& gy,
-    MatType& g)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void Add<InputDataType, OutputDataType>::Backward(
+    const arma::Mat<eT>& /* input */,
+    const arma::Mat<eT>& gy,
+    arma::Mat<eT>& g)
 {
   g = gy;
 }
 
-template<typename MatType>
-void AddType<MatType>::Gradient(
-    const MatType& /* input */,
-    const MatType& error,
-    MatType& gradient)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void Add<InputDataType, OutputDataType>::Gradient(
+    const arma::Mat<eT>& /* input */,
+    const arma::Mat<eT>& error,
+    arma::Mat<eT>& gradient)
 {
   gradient = error;
 }
 
-template<typename MatType>
-void AddType<MatType>::SetWeights(typename MatType::elem_type* weightPtr)
-{
-  // Set the weights to wrap the given memory.
-  MakeAlias(weights, weightPtr, 1, outSize);
-}
-
-template<typename MatType>
-void AddType<MatType>::ComputeOutputDimensions()
-{
-  this->outputDimensions = this->inputDimensions;
-
-  outSize = this->outputDimensions[0];
-  for (size_t i = 1; i < this->outputDimensions.size(); ++i)
-    outSize *= this->outputDimensions[i];
-}
-
-template<typename MatType>
+template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
-void AddType<MatType>::serialize(Archive& ar, const uint32_t /* version */)
+void Add<InputDataType, OutputDataType>::serialize(
+    Archive& ar, const uint32_t /* version */)
 {
-  ar(cereal::base_class<Layer<MatType>>(this));
-
   ar(CEREAL_NVP(outSize));
-  ar(CEREAL_NVP(weights));
+
+  if (cereal::is_loading<Archive>())
+    weights.set_size(outSize, 1);
 }
 
 } // namespace ann

src/mlpack/methods/ann/layer/add_merge.hpp

@@ -1,4 +1,3 @@
-// Temporarily drop.
 /**
  * @file methods/ann/layer/add_merge.hpp
  * @author Marcus Edel
@@ -16,6 +15,10 @@
 
 #include <mlpack/prereqs.hpp>
 
+#include "../visitor/delete_visitor.hpp"
+#include "../visitor/delta_visitor.hpp"
+#include "../visitor/output_parameter_visitor.hpp"
+
 #include "layer_types.hpp"
 
 namespace mlpack {
@@ -25,17 +28,18 @@ namespace ann /** Artificial Neural Network. */ {
  * Implementation of the AddMerge module class. The AddMerge class accumulates
  * the output of various modules.
  *
- * @tparam InputType Type of the input data (arma::colvec, arma::mat,
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
- * @tparam OutputType Type of the output data (arma::colvec, arma::mat,
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
  * @tparam CustomLayers Additional custom layers that can be added.
  */
 template<
-    typename InputType = arma::mat,
-    typename OutputType = arma::mat
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat,
+    typename... CustomLayers
 >
-class AddMerge : public MultiLayer<InputType, OutputType>
+class AddMerge
 {
  public:
   /**
@@ -44,7 +48,7 @@ class AddMerge : public MultiLayer<InputType, OutputType>
    * @param model Expose all the network modules.
    * @param run Call the Forward/Backward method before the output is merged.
    */
-  AddMerge(const bool run = true);
+  AddMerge(const bool model = false, const bool run = true);
 
   /**
    * Create the AddMerge object using the specified parameters.
@@ -53,7 +57,7 @@ class AddMerge : public MultiLayer<InputType, OutputType>
    * @param run Call the Forward/Backward method before the output is merged.
    * @param ownsLayers Delete the layers when this is deallocated.
    */
-  AddMerge(const bool run, const bool ownsLayers);
+  AddMerge(const bool model, const bool run, const bool ownsLayers);
 
   //! Destructor to release allocated memory.
   ~AddMerge();
@@ -65,7 +69,8 @@ class AddMerge : public MultiLayer<InputType, OutputType>
    * @param * (input) Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
-  void Forward(const InputType& input, OutputType& output);
+  template<typename InputType, typename OutputType>
+  void Forward(const InputType& /* input */, OutputType& output);
 
   /**
    * Ordinary feed backward pass of a neural network, calculating the function
@@ -76,9 +81,10 @@ class AddMerge : public MultiLayer<InputType, OutputType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const InputType& /* input */,
-                const OutputType& gy,
-                OutputType& g);
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& /* input */,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g);
 
   /**
    * This is the overload of Backward() that runs only a specific layer with
@@ -89,9 +95,10 @@ class AddMerge : public MultiLayer<InputType, OutputType>
    * @param g The calculated gradient.
    * @param index The index of the layer to run.
    */
-  void Backward(const InputType& /* input */,
-                const OutputType& gy,
-                OutputType& g,
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& /* input */,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g,
                 const size_t index);
 
   /*
@@ -101,9 +108,10 @@ class AddMerge : public MultiLayer<InputType, OutputType>
    * @param error The calculated error.
    * @param gradient The calculated gradient.
    */
-  void Gradient(const InputType& input,
-                const OutputType& error,
-                OutputType& gradient);
+  template<typename eT>
+  void Gradient(const arma::Mat<eT>& input,
+                const arma::Mat<eT>& error,
+                arma::Mat<eT>& gradient);
 
   /*
    * This is the overload of Gradient() that runs a specific layer with the
@@ -114,24 +122,63 @@ class AddMerge : public MultiLayer<InputType, OutputType>
    * @param gradient The calculated gradient.
    * @param The index of the layer to run.
    */
-  void Gradient(const InputType& input,
-                const OutputType& error,
-                OutputType& gradient,
+  template<typename eT>
+  void Gradient(const arma::Mat<eT>& input,
+                const arma::Mat<eT>& error,
+                arma::Mat<eT>& gradient,
                 const size_t index);
 
+  /*
+   * Add a new module to the model.
+   *
+   * @param args The layer parameter.
+   */
+  template <class LayerType, class... Args>
+  void Add(Args... args) { network.push_back(new LayerType(args...)); }
+
+  /*
+   * Add a new module to the model.
+   *
+   * @param layer The Layer to be added to the model.
+   */
+  void Add(LayerTypes<CustomLayers...> layer) { network.push_back(layer); }
+
+  //! Get the input parameter.
+  InputDataType const& InputParameter() const { return inputParameter; }
+  //! Modify the input parameter.
+  InputDataType& InputParameter() { return inputParameter; }
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  //! Return the model modules.
+  std::vector<LayerTypes<CustomLayers...> >& Model()
+  {
+    if (model)
+    {
+      return network;
+    }
+
+    return empty;
+  }
+
+  //! Get the parameters.
+  OutputDataType const& Parameters() const { return weights; }
+  //! Modify the parameters.
+  OutputDataType& Parameters() { return weights; }
+
   //! Get the value of run parameter.
   bool Run() const { return run; }
   //! Modify the value of run parameter.
   bool& Run() { return run; }
 
-  const std::vector<size_t>& OutputDimensions() const
-  {
-    // Propagate input size to child layers.
-    for (size_t i = 0; i < this->network.size(); ++i)
-      this->network[i]->InputDimensions() = this->inputDimensions;
-    return this->network.back()->OutputDimensions();
-  }
-
   /**
    * Serialize the layer.
    */
@@ -139,6 +186,9 @@ class AddMerge : public MultiLayer<InputType, OutputType>
   void serialize(Archive& ar, const uint32_t /* version */);
 
  private:
+  //! Parameter which indicates if the modules should be exposed.
+  bool model;
+
   //! Parameter which indicates if the Forward/Backward method should be called
   //! before merging the output.
   bool run;
@@ -146,6 +196,36 @@ class AddMerge : public MultiLayer<InputType, OutputType>
   //! We need this to know whether we should delete the internally-held layers
   //! in the destructor.
   bool ownsLayers;
+
+  //! Locally-stored network modules.
+  std::vector<LayerTypes<CustomLayers...> > network;
+
+  //! Locally-stored empty list of modules.
+  std::vector<LayerTypes<CustomLayers...> > empty;
+
+  //! Locally-stored delete visitor module object.
+  DeleteVisitor deleteVisitor;
+
+  //! Locally-stored output parameter visitor module object.
+  OutputParameterVisitor outputParameterVisitor;
+
+  //! Locally-stored delta visitor module object.
+  DeltaVisitor deltaVisitor;
+
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored gradient object.
+  OutputDataType gradient;
+
+  //! Locally-stored input parameter object.
+  InputDataType inputParameter;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
+  //! Locally-stored weight object.
+  OutputDataType weights;
 }; // class AddMerge
 
 } // namespace ann

src/mlpack/methods/ann/layer/add_merge_impl.hpp

@@ -0,0 +1,168 @@
+/**
+ * @file methods/ann/layer/add_merge_impl.hpp
+ * @author Marcus Edel
+ *
+ * Definition of the AddMerge module which accumulates the output of the given
+ * modules.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_ADD_MERGE_IMPL_HPP
+#define MLPACK_METHODS_ANN_LAYER_ADD_MERGE_IMPL_HPP
+
+// In case it hasn't yet been included.
+#include "add_merge.hpp"
+
+#include "../visitor/forward_visitor.hpp"
+#include "../visitor/backward_visitor.hpp"
+#include "../visitor/gradient_visitor.hpp"
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+AddMerge<InputDataType, OutputDataType, CustomLayers...>::AddMerge(
+    const bool model, const bool run) :
+    model(model), run(run), ownsLayers(!model)
+{
+  // Nothing to do here.
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+AddMerge<InputDataType, OutputDataType, CustomLayers...>::AddMerge(
+    const bool model, const bool run, const bool ownsLayers) :
+    model(model), run(run), ownsLayers(ownsLayers)
+{
+  // Nothing to do here.
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+AddMerge<InputDataType, OutputDataType, CustomLayers...>::~AddMerge()
+{
+  if (!model && ownsLayers)
+  {
+    std::for_each(network.begin(), network.end(),
+        boost::apply_visitor(deleteVisitor));
+  }
+}
+
+template <typename InputDataType, typename OutputDataType,
+          typename... CustomLayers>
+template<typename InputType, typename OutputType>
+void AddMerge<InputDataType, OutputDataType, CustomLayers...>::Forward(
+    const InputType& input, OutputType& output)
+{
+  if (run)
+  {
+    for (size_t i = 0; i < network.size(); ++i)
+    {
+      boost::apply_visitor(ForwardVisitor(input,
+          boost::apply_visitor(outputParameterVisitor, network[i])),
+          network[i]);
+    }
+  }
+
+  output = boost::apply_visitor(outputParameterVisitor, network.front());
+  for (size_t i = 1; i < network.size(); ++i)
+  {
+    output += boost::apply_visitor(outputParameterVisitor, network[i]);
+  }
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+template<typename eT>
+void AddMerge<InputDataType, OutputDataType, CustomLayers...>::Backward(
+    const arma::Mat<eT>& /* input */,
+    const arma::Mat<eT>& gy,
+    arma::Mat<eT>& g)
+{
+  if (run)
+  {
+    for (size_t i = 0; i < network.size(); ++i)
+    {
+      boost::apply_visitor(BackwardVisitor(boost::apply_visitor(
+          outputParameterVisitor, network[i]), gy,
+          boost::apply_visitor(deltaVisitor, network[i])), network[i]);
+    }
+
+    g = boost::apply_visitor(deltaVisitor, network[0]);
+    for (size_t i = 1; i < network.size(); ++i)
+    {
+      g += boost::apply_visitor(deltaVisitor, network[i]);
+    }
+  }
+  else
+    g = gy;
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+template<typename eT>
+void AddMerge<InputDataType, OutputDataType, CustomLayers...>::Backward(
+    const arma::Mat<eT>& /* input */,
+    const arma::Mat<eT>& gy,
+    arma::Mat<eT>& g,
+    const size_t index)
+{
+  boost::apply_visitor(BackwardVisitor(boost::apply_visitor(
+      outputParameterVisitor, network[index]), gy,
+      boost::apply_visitor(deltaVisitor, network[index])), network[index]);
+  g = boost::apply_visitor(deltaVisitor, network[index]);
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+template<typename eT>
+void AddMerge<InputDataType, OutputDataType, CustomLayers...>::Gradient(
+    const arma::Mat<eT>& input,
+    const arma::Mat<eT>& error,
+    arma::Mat<eT>& /* gradient */ )
+{
+  if (run)
+  {
+    for (size_t i = 0; i < network.size(); ++i)
+    {
+      boost::apply_visitor(GradientVisitor(input, error), network[i]);
+    }
+  }
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+template<typename eT>
+void AddMerge<InputDataType, OutputDataType, CustomLayers...>::Gradient(
+    const arma::Mat<eT>& input,
+    const arma::Mat<eT>& error,
+    arma::Mat<eT>& /* gradient */,
+    const size_t index)
+{
+  boost::apply_visitor(GradientVisitor(input, error), network[index]);
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+template<typename Archive>
+void AddMerge<InputDataType, OutputDataType, CustomLayers...>::serialize(
+    Archive& ar, const uint32_t /* version */)
+{
+  // Be sure to clear other layers before loading.
+  if (cereal::is_loading<Archive>())
+    network.clear();
+
+  ar(CEREAL_VECTOR_VARIANT_POINTER(network));
+  ar(CEREAL_NVP(model));
+  ar(CEREAL_NVP(run));
+  ar(CEREAL_NVP(ownsLayers));
+}
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/layer/alpha_dropout.hpp

@@ -17,7 +17,6 @@
 #define MLPACK_METHODS_ANN_LAYER_ALPHA_DROPOUT_HPP
 
 #include <mlpack/prereqs.hpp>
-#include "layer.hpp"
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
@@ -41,11 +40,14 @@ namespace ann /** Artificial Neural Network. */ {
  * }
  * @endcode
  *
- * @tparam MatType Matrix representation to accept as input and use for
- *    computation.
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
  */
-template<typename MatType = arma::mat>
-class AlphaDropoutType : public Layer<MatType>
+template <typename InputDataType = arma::mat,
+          typename OutputDataType = arma::mat>
+class AlphaDropout
 {
  public:
   /**
@@ -54,33 +56,17 @@ class AlphaDropoutType : public Layer<MatType>
    * @param ratio The probability of setting a value to alphaDash.
    * @param alphaDash The dropout scaling parameter.
    */
-  AlphaDropoutType(const double ratio = 0.5,
-                   const double alphaDash = -alpha * lambda);
+  AlphaDropout(const double ratio = 0.5,
+               const double alphaDash = -alpha * lambda);
 
   /**
-   * Clone the AlphaDropoutType object. This handles polymorphism correctly.
-   */
-  AlphaDropoutType* Clone() const { return new AlphaDropoutType(*this); }
-
-  // Virtual destructor.
-  virtual ~AlphaDropoutType() { }
-
-  //! Copy the given AlphaDropoutType layer.
-  AlphaDropoutType(const AlphaDropoutType& other);
-  //! Take ownership of the given AlphaDropoutType layer.
-  AlphaDropoutType(AlphaDropoutType&& other);
-  //! Copy the given AlphaDropoutType layer.
-  AlphaDropoutType& operator=(const AlphaDropoutType& other);
-  //! Take ownership of the given AlphaDropoutType layer.
-  AlphaDropoutType& operator=(AlphaDropoutType&& other);
-
-  /**
-   * Ordinary feed forward pass of the AlphaDropout layer.
+   * Ordinary feed forward pass of the alpha_dropout layer.
    *
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
-  void Forward(const MatType& input, MatType& output);
+  template<typename eT>
+  void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);
 
   /**
    * Ordinary feed backward pass of the alpha_dropout layer.
@@ -89,7 +75,25 @@ class AlphaDropoutType : public Layer<MatType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const MatType& /* input */, const MatType& gy, MatType& g);
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& /* input */,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g);
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the detla.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  //! The value of the deterministic parameter.
+  bool Deterministic() const { return deterministic; }
+  //! Modify the value of the deterministic parameter.
+  bool& Deterministic() { return deterministic; }
 
   //! The probability of setting a value to alphaDash.
   double Ratio() const { return ratio; }
@@ -101,10 +105,10 @@ class AlphaDropoutType : public Layer<MatType>
   double B() const { return b; }
 
   //! Value of alphaDash.
-  double AlphaDash() const { return alphaDash; }
+  double AlphaDash() const {return alphaDash; }
 
   //! Get the mask.
-  const MatType& Mask() const { return mask; }
+  OutputDataType const& Mask() const {return mask;}
 
   //! Modify the probability of setting a value to alphaDash. As
   //! 'a' and 'b' depend on 'ratio', modify them as well.
@@ -122,8 +126,14 @@ class AlphaDropoutType : public Layer<MatType>
   void serialize(Archive& ar, const uint32_t /* version */);
 
  private:
-  //! Locally-stored mask object.
-  MatType mask;
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
+  //! Locally-stored mast object.
+  OutputDataType mask;
 
   //! The probability of setting a value to aplhaDash.
   double ratio;
@@ -131,6 +141,9 @@ class AlphaDropoutType : public Layer<MatType>
   //! The low variance value of SELU activation function.
   double alphaDash;
 
+  //! If true dropout and scaling is disabled, see notes above.
+  bool deterministic;
+
   //! Value of alpha for normalized inputs (taken from SELU).
   static constexpr double alpha = 1.6732632423543772848170429916717;
 
@@ -142,9 +155,7 @@ class AlphaDropoutType : public Layer<MatType>
 
   //! Value to be added to a*x for affine transformation.
   double b;
-}; // class AlphaDropoutType
-
-typedef AlphaDropoutType<arma::mat> AlphaDropout;
+}; // class AlphaDropout
 
 } // namespace ann
 } // namespace mlpack

src/mlpack/methods/ann/layer/alpha_dropout_impl.hpp

@@ -22,79 +22,25 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template<typename MatType>
-AlphaDropoutType<MatType>::AlphaDropoutType(
+template<typename InputDataType, typename OutputDataType>
+AlphaDropout<InputDataType, OutputDataType>::AlphaDropout(
     const double ratio,
     const double alphaDash) :
     ratio(ratio),
-    alphaDash(alphaDash)
+    alphaDash(alphaDash),
+    deterministic(false)
 {
   Ratio(ratio);
 }
 
-template<typename MatType>
-AlphaDropoutType<MatType>::AlphaDropoutType(const AlphaDropoutType& other) :
-    Layer<MatType>(other),
-    mask(other.mask),
-    ratio(other.ratio),
-    alphaDash(other.alphaDash),
-    a(other.a),
-    b(other.b)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void AlphaDropout<InputDataType, OutputDataType>::Forward(
+    const arma::Mat<eT>& input, arma::Mat<eT>& output)
 {
-  // Nothing to do.
-}
-
-template<typename MatType>
-AlphaDropoutType<MatType>::AlphaDropoutType(AlphaDropoutType&& other) :
-    Layer<MatType>(std::move(other)),
-    mask(std::move(other.mask)),
-    ratio(std::move(other.ratio)),
-    alphaDash(std::move(other.alphaDash)),
-    a(std::move(other.a)),
-    b(std::move(other.b))
-{
-  // Nothing to do.
-}
-
-template<typename MatType>
-AlphaDropoutType<MatType>&
-AlphaDropoutType<MatType>::operator=(const AlphaDropoutType& other)
-{
-  if (&other != this)
-  {
-    Layer<MatType>::operator=(other);
-    mask = other.mask;
-    ratio = other.ratio;
-    alphaDash = other.alphaDash;
-    a = other.a;
-    b = other.b;
-  }
-
-  return *this;
-}
-
-template<typename MatType>
-AlphaDropoutType<MatType>&
-AlphaDropoutType<MatType>::operator=(AlphaDropoutType&& other)
-{
-  if (&other != this)
-  {
-    Layer<MatType>::operator=(std::move(other));
-    mask = std::move(other.mask);
-    ratio = std::move(other.ratio);
-    alphaDash = std::move(other.alphaDash);
-    a = std::move(other.a);
-    b = std::move(other.b);
-  }
-
-  return *this;
-}
-
-template<typename MatType>
-void AlphaDropoutType<MatType>::Forward(const MatType& input, MatType& output)
-{
-  // The dropout mask will not be multiplied during testing.
-  if (!this->training)
+  // The dropout mask will not be multiplied in the deterministic mode
+  // (during testing).
+  if (deterministic)
   {
     output = input;
   }
@@ -103,35 +49,29 @@ void AlphaDropoutType<MatType>::Forward(const MatType& input, MatType& output)
     // Set values to alphaDash with probability ratio.  Then apply affine
     // transformation so as to keep mean and variance of outputs to their
     // original values.
-    mask = arma::randu<MatType>(input.n_rows, input.n_cols);
+    mask = arma::randu< arma::Mat<eT> >(input.n_rows, input.n_cols);
     mask.transform( [&](double val) { return (val > ratio); } );
     output = (input % mask + alphaDash * (1 - mask)) * a + b;
   }
 }
 
-template<typename MatType>
-void AlphaDropoutType<MatType>::Backward(
-    const MatType& /* input */, const MatType& gy, MatType& g)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void AlphaDropout<InputDataType, OutputDataType>::Backward(
+    const arma::Mat<eT>& /* input */, const arma::Mat<eT>& gy, arma::Mat<eT>& g)
 {
   g = gy % mask * a;
 }
 
-template<typename MatType>
+template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
-void AlphaDropoutType<MatType>::serialize(
+void AlphaDropout<InputDataType, OutputDataType>::serialize(
     Archive& ar, const uint32_t /* version */)
 {
-  ar(cereal::base_class<Layer<MatType>>(this));
-
   ar(CEREAL_NVP(ratio));
   ar(CEREAL_NVP(alphaDash));
   ar(CEREAL_NVP(a));
   ar(CEREAL_NVP(b));
-
-  // No need to serialize the mask, since it will be recomputed on the next
-  // forward pass.  But we should clear it if we are loading.
-  if (Archive::is_loading::value)
-    mask.clear();
 }
 
 } // namespace ann

src/mlpack/methods/ann/layer/atrous_convolution.hpp

@@ -1,4 +1,3 @@
-// Temporarily drop.
 /**
  * @file methods/ann/layer/atrous_convolution.hpp
  * @author Aarush Gupta
@@ -47,10 +46,10 @@ template <
     typename ForwardConvolutionRule = NaiveConvolution<ValidConvolution>,
     typename BackwardConvolutionRule = NaiveConvolution<FullConvolution>,
     typename GradientConvolutionRule = NaiveConvolution<ValidConvolution>,
-    typename InputType = arma::mat,
-    typename OutputType = arma::mat
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
 >
-class AtrousConvolution : public Layer<InputType, OutputType>
+class AtrousConvolution
 {
  public:
   //! Create the AtrousConvolution object.
@@ -140,7 +139,8 @@ class AtrousConvolution : public Layer<InputType, OutputType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
-  void Forward(const InputType& input, OutputType& output);
+  template<typename eT>
+  void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);
 
   /**
    * Ordinary feed backward pass of a neural network, calculating the function
@@ -151,9 +151,10 @@ class AtrousConvolution : public Layer<InputType, OutputType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const InputType& /* input */,
-                const OutputType& gy,
-                OutputType& g);
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& /* input */,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g);
 
   /*
    * Calculate the gradient using the output delta and the input activation.
@@ -162,90 +163,101 @@ class AtrousConvolution : public Layer<InputType, OutputType>
    * @param error The calculated error.
    * @param gradient The calculated gradient.
    */
-  void Gradient(const InputType& /* input */,
-                const OutputType& error,
-                OutputType& gradient);
+  template<typename eT>
+  void Gradient(const arma::Mat<eT>& /* input */,
+                const arma::Mat<eT>& error,
+                arma::Mat<eT>& gradient);
 
   //! Get the parameters.
-  OutputType const& Parameters() const { return weights; }
+  OutputDataType const& Parameters() const { return weights; }
   //! Modify the parameters.
-  OutputType& Parameters() { return weights; }
+  OutputDataType& Parameters() { return weights; }
 
   //! Get the weight of the layer.
-  const arma::Cube<typename OutputType::elem_type>& Weight() const
-  {
-    return weight;
-  }
+  arma::cube const& Weight() const { return weight; }
   //! Modify the weight of the layer.
-  arma::Cube<typename OutputType::elem_type>& Weight() { return weight; }
-
-  const std::vector<size_t>& OutputDimensions() const
-  {
-    std::vector<size_t> result(inputDimensions.size(), 0);
-    result[0] = outputWidth;
-    result[1] = outputHeight;
-    return result;
-  }
+  arma::cube& Weight() { return weight; }
 
   //! Get the bias of the layer.
-  const OutputType& Bias() const { return bias; }
+  arma::mat const& Bias() const { return bias; }
   //! Modify the bias of the layer.
-  OutputType& Bias() { return bias; }
+  arma::mat& Bias() { return bias; }
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  //! Get the gradient.
+  OutputDataType const& Gradient() const { return gradient; }
+  //! Modify the gradient.
+  OutputDataType& Gradient() { return gradient; }
 
   //! Get the input width.
-  const size_t& InputWidth() const { return inputWidth; }
+  size_t InputWidth() const { return inputWidth; }
   //! Modify input the width.
   size_t& InputWidth() { return inputWidth; }
 
   //! Get the input height.
-  const size_t& InputHeight() const { return inputHeight; }
+  size_t InputHeight() const { return inputHeight; }
   //! Modify the input height.
   size_t& InputHeight() { return inputHeight; }
 
   //! Get the output width.
-  const size_t& OutputWidth() const { return outputWidth; }
+  size_t OutputWidth() const { return outputWidth; }
   //! Modify the output width.
   size_t& OutputWidth() { return outputWidth; }
 
   //! Get the output height.
-  const size_t& OutputHeight() const { return outputHeight; }
+  size_t OutputHeight() const { return outputHeight; }
   //! Modify the output height.
   size_t& OutputHeight() { return outputHeight; }
 
+  //! Get the input size.
+  size_t InputSize() const { return inSize; }
+
+  //! Get the output size.
+  size_t OutputSize() const { return outSize; }
+
   //! Get the kernel width.
-  const size_t& KernelWidth() const { return kernelWidth; }
+  size_t KernelWidth() const { return kernelWidth; }
   //! Modify the kernel width.
   size_t& KernelWidth() { return kernelWidth; }
 
   //! Get the kernel height.
-  const size_t& KernelHeight() const { return kernelHeight; }
+  size_t KernelHeight() const { return kernelHeight; }
   //! Modify the kernel height.
   size_t& KernelHeight() { return kernelHeight; }
 
   //! Get the stride width.
-  const size_t& StrideWidth() const { return strideWidth; }
+  size_t StrideWidth() const { return strideWidth; }
   //! Modify the stride width.
   size_t& StrideWidth() { return strideWidth; }
 
   //! Get the stride height.
-  const size_t& StrideHeight() const { return strideHeight; }
+  size_t StrideHeight() const { return strideHeight; }
   //! Modify the stride height.
   size_t& StrideHeight() { return strideHeight; }
 
   //! Get the dilation rate on the X axis.
-  const size_t& DilationWidth() const { return dilationWidth; }
+  size_t DilationWidth() const { return dilationWidth; }
   //! Modify the dilation rate on the X axis.
   size_t& DilationWidth() { return dilationWidth; }
 
   //! Get the dilation rate on the Y axis.
-  const size_t& DilationHeight() const { return dilationHeight; }
+  size_t DilationHeight() const { return dilationHeight; }
   //! Modify the dilation rate on the Y axis.
   size_t& DilationHeight() { return dilationHeight; }
 
   //! Get the internal Padding layer.
-  PaddingType<InputType, OutputType> const& Padding() const { return padding; }
+  ann::Padding<> const& Padding() const { return padding; }
   //! Modify the internal Padding layer.
-  PaddingType<InputType, OutputType>& Padding() { return padding; }
+  ann::Padding<>& Padding() { return padding; }
 
   //! Get size of the weight matrix.
   size_t WeightSize() const
@@ -346,13 +358,13 @@ class AtrousConvolution : public Layer<InputType, OutputType>
   size_t strideHeight;
 
   //! Locally-stored weight object.
-  OutputType weights;
+  OutputDataType weights;
 
   //! Locally-stored weight object.
-  arma::Cube<typename OutputType::elem_type> weight;
+  arma::cube weight;
 
   //! Locally-stored bias term object.
-  OutputType bias;
+  arma::mat bias;
 
   //! Locally-stored input width.
   size_t inputWidth;
@@ -373,19 +385,28 @@ class AtrousConvolution : public Layer<InputType, OutputType>
   size_t dilationHeight;
 
   //! Locally-stored transformed output parameter.
-  arma::Cube<typename OutputType::elem_type> outputTemp;
+  arma::cube outputTemp;
 
   //! Locally-stored transformed padded input parameter.
-  arma::Cube<typename OutputType::elem_type> inputPaddedTemp;
+  arma::cube inputPaddedTemp;
 
   //! Locally-stored transformed error parameter.
-  arma::Cube<typename OutputType::elem_type> gTemp;
+  arma::cube gTemp;
 
   //! Locally-stored transformed gradient parameter.
-  arma::Cube<typename OutputType::elem_type> gradientTemp;
+  arma::cube gradientTemp;
 
   //! Locally-stored padding layer.
-  PaddingType<InputType, OutputType> padding;
+  ann::Padding<> padding;
+
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored gradient object.
+  OutputDataType gradient;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
 }; // class AtrousConvolution
 
 } // namespace ann

src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp

@@ -23,15 +23,15 @@ template<
     typename ForwardConvolutionRule,
     typename BackwardConvolutionRule,
     typename GradientConvolutionRule,
-    typename InputType,
-    typename OutputType
+    typename InputDataType,
+    typename OutputDataType
 >
 AtrousConvolution<
     ForwardConvolutionRule,
     BackwardConvolutionRule,
     GradientConvolutionRule,
-    InputType,
-    OutputType
+    InputDataType,
+    OutputDataType
 >::AtrousConvolution()
 {
   // Nothing to do here.
@@ -41,15 +41,15 @@ template<
     typename ForwardConvolutionRule,
     typename BackwardConvolutionRule,
     typename GradientConvolutionRule,
-    typename InputType,
-    typename OutputType
+    typename InputDataType,
+    typename OutputDataType
 >
 AtrousConvolution<
     ForwardConvolutionRule,
     BackwardConvolutionRule,
     GradientConvolutionRule,
-    InputType,
-    OutputType
+    InputDataType,
+    OutputDataType
 >::AtrousConvolution(
     const size_t inSize,
     const size_t outSize,
@@ -86,15 +86,15 @@ template<
     typename ForwardConvolutionRule,
     typename BackwardConvolutionRule,
     typename GradientConvolutionRule,
-    typename InputType,
-    typename OutputType
+    typename InputDataType,
+    typename OutputDataType
 >
 AtrousConvolution<
     ForwardConvolutionRule,
     BackwardConvolutionRule,
     GradientConvolutionRule,
-    InputType,
-    OutputType
+    InputDataType,
+    OutputDataType
 >::AtrousConvolution(
     const size_t inSize,
     const size_t outSize,
@@ -143,49 +143,49 @@ AtrousConvolution<
     InitializeSamePadding(padWLeft, padWRight, padHTop, padHBottom);
   }
 
-  padding = PaddingType<InputType, OutputType>(padWLeft, padWRight, padHTop,
-      padHBottom);
+  padding = ann::Padding<>(padWLeft, padWRight, padHTop, padHBottom);
 }
 
 template<
     typename ForwardConvolutionRule,
     typename BackwardConvolutionRule,
     typename GradientConvolutionRule,
-    typename InputType,
-    typename OutputType
+    typename InputDataType,
+    typename OutputDataType
 >
 void AtrousConvolution<
     ForwardConvolutionRule,
     BackwardConvolutionRule,
     GradientConvolutionRule,
-    InputType,
-    OutputType
->::ResetWeights(typename OutputType::elem_type* weightsPtr)
+    InputDataType,
+    OutputDataType
+>::Reset()
 {
-  weight = arma::Cube<typename OutputType::elem_type>(weightsPtr, kernelWidth,
-      kernelHeight, outSize * inSize, false, true);
-  bias = OutputType(weightsPtr + weight.n_elem, outSize, 1, false, true);
+    weight = arma::cube(weights.memptr(), kernelWidth, kernelHeight,
+        outSize * inSize, false, false);
+    bias = arma::mat(weights.memptr() + weight.n_elem,
+        outSize, 1, false, false);
 }
 
 template<
     typename ForwardConvolutionRule,
     typename BackwardConvolutionRule,
     typename GradientConvolutionRule,
-    typename InputType,
-    typename OutputType
+    typename InputDataType,
+    typename OutputDataType
 >
+template<typename eT>
 void AtrousConvolution<
     ForwardConvolutionRule,
     BackwardConvolutionRule,
     GradientConvolutionRule,
-    InputType,
-    OutputType
->::Forward(const InputType& input, OutputType& output)
+    InputDataType,
+    OutputDataType
+>::Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output)
 {
   batchSize = input.n_cols;
-  arma::Cube<typename InputType::elem_type> inputTemp(
-      const_cast<InputType&>(input).memptr(), inputWidth, inputHeight, inSize *
-      batchSize, false, false);
+  arma::cube inputTemp(const_cast<arma::Mat<eT>&>(input).memptr(),
+      inputWidth, inputHeight, inSize * batchSize, false, false);
 
   if (padding.PadWLeft() != 0 || padding.PadWRight() != 0 ||
       padding.PadHTop() != 0 || padding.PadHBottom() != 0)
@@ -207,8 +207,8 @@ void AtrousConvolution<
       padding.PadHTop(), padding.PadHBottom(), dilationHeight);
 
   output.set_size(wConv * hConv * outSize, batchSize);
-  outputTemp = arma::Cube<typename OutputType::elem_type>(output.memptr(),
-      wConv, hConv, outSize * batchSize, false, false);
+  outputTemp = arma::Cube<eT>(output.memptr(), wConv, hConv,
+      outSize * batchSize, false, false);
   outputTemp.zeros();
 
   for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap <
@@ -222,7 +222,7 @@ void AtrousConvolution<
 
     for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++)
     {
-      OutputType convOutput;
+      arma::Mat<eT> convOutput;
 
       if (padding.PadWLeft() != 0 || padding.PadWRight() != 0 ||
           padding.PadHTop() != 0 || padding.PadHBottom() != 0)
@@ -252,24 +252,25 @@ template<
     typename ForwardConvolutionRule,
     typename BackwardConvolutionRule,
     typename GradientConvolutionRule,
-    typename InputType,
-    typename OutputType
+    typename InputDataType,
+    typename OutputDataType
 >
+template<typename eT>
 void AtrousConvolution<
     ForwardConvolutionRule,
     BackwardConvolutionRule,
     GradientConvolutionRule,
-    InputType,
-    OutputType
->::Backward(const InputType& /* input */, const OutputType& gy, OutputType& g)
+    InputDataType,
+    OutputDataType
+>::Backward(
+    const arma::Mat<eT>& /* input */, const arma::Mat<eT>& gy, arma::Mat<eT>& g)
 {
-  arma::Cube<typename OutputType::elem_type> mappedError(
-      ((OutputType&) gy).memptr(), outputWidth, outputHeight, outSize *
-      batchSize, false, false);
+  arma::cube mappedError(((arma::Mat<eT>&) gy).memptr(), outputWidth,
+      outputHeight, outSize * batchSize, false, false);
 
   g.set_size(inputWidth * inputHeight * inSize, batchSize);
-  gTemp = arma::Cube<typename OutputType::elem_type>(g.memptr(), inputWidth,
-      inputHeight, inSize * batchSize, false, false);
+  gTemp = arma::Cube<eT>(g.memptr(), inputWidth, inputHeight,
+      inSize * batchSize, false, false);
   gTemp.zeros();
 
   for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap <
@@ -283,7 +284,7 @@ void AtrousConvolution<
 
     for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++)
     {
-      OutputType output, rotatedFilter;
+      arma::Mat<eT> output, rotatedFilter;
       Rotate180(weight.slice(outMapIdx), rotatedFilter);
 
       BackwardConvolutionRule::Convolution(mappedError.slice(outMap),
@@ -310,30 +311,29 @@ template<
     typename ForwardConvolutionRule,
     typename BackwardConvolutionRule,
     typename GradientConvolutionRule,
-    typename InputType,
-    typename OutputType
+    typename InputDataType,
+    typename OutputDataType
 >
+template<typename eT>
 void AtrousConvolution<
     ForwardConvolutionRule,
     BackwardConvolutionRule,
     GradientConvolutionRule,
-    InputType,
-    OutputType
+    InputDataType,
+    OutputDataType
 >::Gradient(
-    const InputType& input,
-    const OutputType& error,
-    OutputType& gradient)
+    const arma::Mat<eT>& input,
+    const arma::Mat<eT>& error,
+    arma::Mat<eT>& gradient)
 {
-  arma::Cube<typename OutputType::elem_type> mappedError(
-      ((OutputType&) error).memptr(), outputWidth, outputHeight, outSize *
-      batchSize, false, false);
-  arma::Cube<typename InputType::elem_type> inputTemp(
-      const_cast<InputType&>(input).memptr(), inputWidth, inputHeight,
-      inSize * batchSize, false, false);
+  arma::cube mappedError(((arma::Mat<eT>&) error).memptr(), outputWidth,
+      outputHeight, outSize * batchSize, false, false);
+  arma::cube inputTemp(const_cast<arma::Mat<eT>&>(input).memptr(),
+      inputWidth, inputHeight, inSize * batchSize, false, false);
 
   gradient.set_size(weights.n_elem, 1);
-  gradientTemp = arma::Cube<typename OutputType::elem_type>(gradient.memptr(),
-      weight.n_rows, weight.n_cols, weight.n_slices, false, false);
+  gradientTemp = arma::Cube<eT>(gradient.memptr(), weight.n_rows,
+      weight.n_cols, weight.n_slices, false, false);
   gradientTemp.zeros();
 
   for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap <
@@ -347,7 +347,7 @@ void AtrousConvolution<
 
     for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++)
     {
-      InputType inputSlice;
+      arma::Mat<eT> inputSlice;
       if (padding.PadWLeft() != 0 || padding.PadWRight() != 0 ||
           padding.PadHTop() != 0 || padding.PadHBottom() != 0)
       {
@@ -358,9 +358,9 @@ void AtrousConvolution<
         inputSlice = inputTemp.slice(inMap + batchCount * inSize);
       }
 
-      OutputType deltaSlice = mappedError.slice(outMap);
+      arma::Mat<eT> deltaSlice = mappedError.slice(outMap);
 
-      OutputType output;
+      arma::Mat<eT> output;
       GradientConvolutionRule::Convolution(inputSlice, deltaSlice,
           output, strideWidth, strideHeight, 1, 1);
 
@@ -404,20 +404,18 @@ template<
     typename ForwardConvolutionRule,
     typename BackwardConvolutionRule,
     typename GradientConvolutionRule,
-    typename InputType,
-    typename OutputType
+    typename InputDataType,
+    typename OutputDataType
 >
 template<typename Archive>
 void AtrousConvolution<
     ForwardConvolutionRule,
     BackwardConvolutionRule,
     GradientConvolutionRule,
-    InputType,
-    OutputType
+    InputDataType,
+    OutputDataType
 >::serialize(Archive& ar, const uint32_t /* version */)
 {
-  ar(cereal::base_class<Layer<InputType, OutputType>>(this));
-
   ar(CEREAL_NVP(inSize));
   ar(CEREAL_NVP(outSize));
   ar(CEREAL_NVP(batchSize));
@@ -432,22 +430,27 @@ void AtrousConvolution<
   ar(CEREAL_NVP(dilationWidth));
   ar(CEREAL_NVP(dilationHeight));
   ar(CEREAL_NVP(padding));
-  ar(CEREAL_NVP(weights));
+
+  if (cereal::is_loading<Archive>())
+  {
+    weights.set_size((outSize * inSize * kernelWidth * kernelHeight) + outSize,
+        1);
+  }
 }
 
 template<
     typename ForwardConvolutionRule,
     typename BackwardConvolutionRule,
     typename GradientConvolutionRule,
-    typename InputType,
-    typename OutputType
+    typename InputDataType,
+    typename OutputDataType
 >
 void AtrousConvolution<
     ForwardConvolutionRule,
     BackwardConvolutionRule,
     GradientConvolutionRule,
-    InputType,
-    OutputType
+    InputDataType,
+    OutputDataType
 >::InitializeSamePadding(size_t& padWLeft,
                          size_t& padWRight,
                          size_t& padHTop,

src/mlpack/methods/ann/layer/base_layer.hpp

@@ -30,7 +30,6 @@
 #include <mlpack/methods/ann/activation_functions/hard_swish_function.hpp>
 #include <mlpack/methods/ann/activation_functions/tanh_exponential_function.hpp>
 #include <mlpack/methods/ann/activation_functions/silu_function.hpp>
-#include "layer.hpp"
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
@@ -41,83 +40,102 @@ namespace ann /** Artificial Neural Network. */ {
  *
  * A few convenience typedefs are given:
  *
- *  - Sigmoid
- *  - ReLU
- *  - TanH
- *  - Softplus
- *  - HardSigmoid
- *  - Swish
- *  - Mish
- *  - LiSHT
- *  - GELU
- *  - ELiSH
- *  - Elliot
- *  - Gaussian
- *  - HardSwish
- *  - TanhExp
- *  - SILU
+ *  - SigmoidLayer
+ *  - IdentityLayer
+ *  - ReLULayer
+ *  - TanHLayer
+ *  - SoftplusLayer
+ *  - HardSigmoidLayer
+ *  - SwishLayer
+ *  - MishLayer
+ *  - LiSHTLayer
+ *  - GELULayer
+ *  - ELiSHLayer
+ *  - ElliotLayer
+ *  - GaussianLayer
+ *  - HardSwishLayer
+ *  - TanhExpLayer
+ *  - SILULayer
  *
  * @tparam ActivationFunction Activation function used for the embedding layer.
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
  */
 template <
     class ActivationFunction = LogisticFunction,
-    typename MatType = arma::mat
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
 >
-class BaseLayer : public Layer<MatType>
+class BaseLayer
 {
  public:
   /**
    * Create the BaseLayer object.
    */
-  BaseLayer() : Layer<MatType>()
+  BaseLayer()
   {
     // Nothing to do here.
   }
 
-  // Virtual destructor.
-  virtual ~BaseLayer() { }
-
-  // No copy constructor or operators needed here, since the class has no
-  // members.
-
-  //! Clone the BaseLayer object. This handles polymorphism correctly.
-  BaseLayer* Clone() const { return new BaseLayer(*this); }
-
   /**
-   * Forward pass: apply the activation to the inputs.
+   * Ordinary feed forward pass of a neural network, evaluating the function
+   * f(x) by propagating the activity forward through f.
    *
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
-  void Forward(const MatType& input, MatType& output)
+  template<typename InputType, typename OutputType>
+  void Forward(const InputType& input, OutputType& output)
   {
     ActivationFunction::Fn(input, output);
   }
 
   /**
-   * Backward pass: compute the function f(x) by propagating x backwards through
-   * f, using the results from the forward pass.
+   * Ordinary feed backward pass of a neural network, calculating the function
+   * f(x) by propagating x backwards trough f. Using the results from the feed
+   * forward pass.
    *
    * @param input The propagated input activation.
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const MatType& input, const MatType& gy, MatType& g)
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& input,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g)
   {
-    MatType derivative;
+    arma::Mat<eT> derivative;
     ActivationFunction::Deriv(input, derivative);
     g = gy % derivative;
   }
 
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
   /**
    * Serialize the layer.
    */
   template<typename Archive>
-  void serialize(Archive& ar, const uint32_t /* version */)
+  void serialize(Archive& /* ar */, const uint32_t /* version */)
   {
-    ar(cereal::base_class<Layer<MatType>>(this));
-    // Nothing to serialize.
+    /* Nothing to do here */
   }
+
+ private:
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
 }; // class BaseLayer
 
 // Convenience typedefs.
@@ -125,122 +143,179 @@ class BaseLayer : public Layer<MatType>
 /**
  * Standard Sigmoid-Layer using the logistic activation function.
  */
-typedef BaseLayer<LogisticFunction, arma::mat> Sigmoid;
+template <
+    class ActivationFunction = LogisticFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using SigmoidLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
 
-template<typename MatType = arma::mat>
-using SigmoidType = BaseLayer<LogisticFunction, MatType>;
+/**
+ * Standard Identity-Layer using the identity activation function.
+ */
+template <
+    class ActivationFunction = IdentityFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using IdentityLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
 
 /**
  * Standard rectified linear unit non-linearity layer.
  */
-typedef BaseLayer<RectifierFunction, arma::mat> ReLU;
-
-template<typename MatType = arma::mat>
-using ReLUType = BaseLayer<RectifierFunction, MatType>;
+template <
+    class ActivationFunction = RectifierFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using ReLULayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
 
 /**
  * Standard hyperbolic tangent layer.
  */
-typedef BaseLayer<TanhFunction, arma::mat> TanH;
-
-template<typename MatType = arma::mat>
-using TanHType = BaseLayer<TanhFunction, MatType>;
+template <
+    class ActivationFunction = TanhFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using TanHLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
 
 /**
  * Standard Softplus-Layer using the Softplus activation function.
  */
-typedef BaseLayer<SoftplusFunction, arma::mat> SoftPlus;
-
-template<typename MatType = arma::mat>
-using SoftPlusType = BaseLayer<SoftplusFunction, MatType>;
+template <
+    class ActivationFunction = SoftplusFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using SoftPlusLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
 
 /**
  * Standard HardSigmoid-Layer using the HardSigmoid activation function.
  */
-typedef BaseLayer<HardSigmoidFunction, arma::mat> HardSigmoid;
-
-template<typename MatType = arma::mat>
-using HardSigmoidType = BaseLayer<HardSigmoidFunction, MatType>;
+template <
+    class ActivationFunction = HardSigmoidFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using HardSigmoidLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
 
 /**
  * Standard Swish-Layer using the Swish activation function.
  */
-typedef BaseLayer<SwishFunction, arma::mat> Swish;
-
-template<typename MatType = arma::mat>
-using SwishType = BaseLayer<SwishFunction, MatType>;
+template <
+    class ActivationFunction = SwishFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using SwishFunctionLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
 
 /**
  * Standard Mish-Layer using the Mish activation function.
  */
-typedef BaseLayer<MishFunction, arma::mat> Mish;
-
-template<typename MatType = arma::mat>
-using MishType = BaseLayer<MishFunction, MatType>;
+template <
+    class ActivationFunction = MishFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using MishFunctionLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
 
 /**
  * Standard LiSHT-Layer using the LiSHT activation function.
  */
-typedef BaseLayer<LiSHTFunction, arma::mat> LiSHT;
-
-template<typename MatType = arma::mat>
-using LiSHTType = BaseLayer<LiSHTFunction, MatType>;
+template <
+    class ActivationFunction = LiSHTFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using LiSHTFunctionLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
 
 /**
  * Standard GELU-Layer using the GELU activation function.
  */
-typedef BaseLayer<GELUFunction, arma::mat> GELU;
-
-template<typename MatType = arma::mat>
-using GELUType = BaseLayer<GELUFunction, MatType>;
+template <
+    class ActivationFunction = GELUFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using GELUFunctionLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
 
 /**
  * Standard Elliot-Layer using the Elliot activation function.
  */
-typedef BaseLayer<ElliotFunction, arma::mat> Elliot;
-
-template<typename MatType = arma::mat>
-using ElliotType = BaseLayer<ElliotFunction, MatType>;
+template <
+    class ActivationFunction = ElliotFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using ElliotFunctionLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
 
 /**
  * Standard ELiSH-Layer using the ELiSH activation function.
  */
-typedef BaseLayer<ElishFunction, arma::mat> Elish;
-
-template<typename MatType = arma::mat>
-using ElishType = BaseLayer<ElishFunction, MatType>;
+template <
+    class ActivationFunction = ElishFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using ElishFunctionLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
 
 /**
  * Standard Gaussian-Layer using the Gaussian activation function.
  */
-typedef BaseLayer<GaussianFunction, arma::mat> Gaussian;
-
-template<typename MatType = arma::mat>
-using GaussianType = BaseLayer<GaussianFunction, MatType>;
+template <
+    class ActivationFunction = GaussianFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using GaussianFunctionLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
 
 /**
  * Standard HardSwish-Layer using the HardSwish activation function.
  */
-typedef BaseLayer<HardSwishFunction, arma::mat> HardSwish;
-
-template <typename MatType = arma::mat>
-using HardSwishType = BaseLayer<HardSwishFunction, MatType>;
+template <
+    class ActivationFunction = HardSwishFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using HardSwishFunctionLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
 
 /**
  * Standard TanhExp-Layer using the TanhExp activation function.
  */
-typedef BaseLayer<TanhExpFunction, arma::mat> TanhExp;
-
-template<typename MatType = arma::mat>
-using TanhExpType = BaseLayer<TanhExpFunction, MatType>;
+template <
+    class ActivationFunction = TanhExpFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using TanhExpFunctionLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
 
 /**
  * Standard SILU-Layer using the SILU activation function.
  */
-typedef BaseLayer<SILUFunction, arma::mat> SILU;
-
-template<typename MatType = arma::mat>
-using SILUType = BaseLayer<SILUFunction, MatType>;
+template <
+    class ActivationFunction = SILUFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using SILUFunctionLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType
+>;
 
 } // namespace ann
 } // namespace mlpack

src/mlpack/methods/ann/layer/batch_norm.hpp

@@ -50,10 +50,10 @@ namespace ann /** Artificial Neural Network. */ {
  *         arma::sp_mat or arma::cube).
  */
 template <
-  typename InputType = arma::mat,
-  typename OutputType = arma::mat
+  typename InputDataType = arma::mat,
+  typename OutputDataType = arma::mat
 >
-class BatchNorm : public Layer<InputType, OutputType>
+class BatchNorm
 {
  public:
   //! Create the BatchNorm object.
@@ -74,9 +74,9 @@ class BatchNorm : public Layer<InputType, OutputType>
             const double momentum = 0.1);
 
   /**
-   * Reset the layer parameters.
+   * Reset the layer parameters
    */
-  void SetWeights(typename OutputType::elem_type* weightsPtr);
+  void Reset();
 
   /**
    * Forward pass of the Batch Normalization layer. Transforms the input data
@@ -86,7 +86,8 @@ class BatchNorm : public Layer<InputType, OutputType>
    * @param input Input data for the layer
    * @param output Resulting output activations.
    */
-  void Forward(const InputType& input, OutputType& output);
+  template<typename eT>
+  void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);
 
   /**
    * Backward pass through the layer.
@@ -95,9 +96,10 @@ class BatchNorm : public Layer<InputType, OutputType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const InputType& input,
-                const OutputType& gy,
-                OutputType& g);
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& input,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g);
 
   /**
    * Calculate the gradient using the output delta and the input activations.
@@ -106,24 +108,45 @@ class BatchNorm : public Layer<InputType, OutputType>
    * @param error The calculated error
    * @param gradient The calculated gradient.
    */
-  void Gradient(const InputType& input,
-                const OutputType& error,
-                OutputType& gradient);
+  template<typename eT>
+  void Gradient(const arma::Mat<eT>& input,
+                const arma::Mat<eT>& error,
+                arma::Mat<eT>& gradient);
 
   //! Get the parameters.
-  const OutputType& Parameters() const { return weights; }
+  OutputDataType const& Parameters() const { return weights; }
   //! Modify the parameters.
-  OutputType& Parameters() { return weights; }
+  OutputDataType& Parameters() { return weights; }
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  //! Get the gradient.
+  OutputDataType const& Gradient() const { return gradient; }
+  //! Modify the gradient.
+  OutputDataType& Gradient() { return gradient; }
+
+  //! Get the value of deterministic parameter.
+  bool Deterministic() const { return deterministic; }
+  //! Modify the value of deterministic parameter.
+  bool& Deterministic() { return deterministic; }
 
   //! Get the mean over the training data.
-  const OutputType& TrainingMean() const { return runningMean; }
+  OutputDataType const& TrainingMean() const { return runningMean; }
   //! Modify the mean over the training data.
-  OutputType& TrainingMean() { return runningMean; }
+  OutputDataType& TrainingMean() { return runningMean; }
 
   //! Get the variance over the training data.
-  const OutputType& TrainingVariance() const { return runningVariance; }
+  OutputDataType const& TrainingVariance() const { return runningVariance; }
   //! Modify the variance over the training data.
-  OutputType& TrainingVariance() { return runningVariance; }
+  OutputDataType& TrainingVariance() { return runningVariance; }
 
   //! Get the number of input units / channels.
   size_t InputSize() const { return size; }
@@ -164,19 +187,25 @@ class BatchNorm : public Layer<InputType, OutputType>
   bool loading;
 
   //! Locally-stored scale parameter.
-  OutputType gamma;
+  OutputDataType gamma;
 
   //! Locally-stored shift parameter.
-  OutputType beta;
+  OutputDataType beta;
 
   //! Locally-stored mean object.
-  OutputType mean;
+  OutputDataType mean;
 
   //! Locally-stored variance object.
-  OutputType variance;
+  OutputDataType variance;
 
   //! Locally-stored parameters.
-  OutputType weights;
+  OutputDataType weights;
+
+  /**
+   * If true then mean and variance over the training set will be considered
+   * instead of being calculated over the batch.
+   */
+  bool deterministic;
 
   //! Locally-stored running mean/variance counter.
   size_t count;
@@ -186,16 +215,25 @@ class BatchNorm : public Layer<InputType, OutputType>
   double averageFactor;
 
   //! Locally-stored mean object.
-  OutputType runningMean;
+  OutputDataType runningMean;
 
   //! Locally-stored variance object.
-  OutputType runningVariance;
+  OutputDataType runningVariance;
+
+  //! Locally-stored gradient object.
+  OutputDataType gradient;
+
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
 
   //! Locally-stored normalized input.
-  arma::Cube<typename InputType::elem_type> normalized;
+  arma::cube normalized;
 
   //! Locally-stored zero mean input.
-  arma::Cube<typename InputType::elem_type> inputMean;
+  arma::cube inputMean;
 }; // class BatchNorm
 
 } // namespace ann

src/mlpack/methods/ann/layer/batch_norm_impl.hpp

@@ -21,21 +21,22 @@
 namespace mlpack {
 namespace ann { /** Artificial Neural Network. */
 
-template<typename InputType, typename OutputType>
-BatchNorm<InputType, OutputType>::BatchNorm() :
+template<typename InputDataType, typename OutputDataType>
+BatchNorm<InputDataType, OutputDataType>::BatchNorm() :
     size(0),
     eps(1e-8),
     average(true),
     momentum(0.0),
     loading(false),
+    deterministic(false),
     count(0),
     averageFactor(0.0)
 {
   // Nothing to do here.
 }
 
-template <typename InputType, typename OutputType>
-BatchNorm<InputType, OutputType>::BatchNorm(
+template <typename InputDataType, typename OutputDataType>
+BatchNorm<InputDataType, OutputDataType>::BatchNorm(
     const size_t size,
     const double eps,
     const bool average,
@@ -45,6 +46,7 @@ BatchNorm<InputType, OutputType>::BatchNorm(
     average(average),
     momentum(momentum),
     loading(false),
+    deterministic(false),
     count(0),
     averageFactor(0.0)
 {
@@ -53,14 +55,13 @@ BatchNorm<InputType, OutputType>::BatchNorm(
   runningVariance.ones(size, 1);
 }
 
-template<typename InputType, typename OutputType>
-void BatchNorm<InputType, OutputType>::SetWeights(
-    typename OutputType::elem_type* weightsPtr)
+template<typename InputDataType, typename OutputDataType>
+void BatchNorm<InputDataType, OutputDataType>::Reset()
 {
   // Gamma acts as the scaling parameters for the normalized output.
-  gamma = OutputType(weightsPtr, size, 1, false, false);
+  gamma = arma::mat(weights.memptr(), size, 1, false, false);
   // Beta acts as the shifting parameters for the normalized output.
-  beta = OutputType(weightsPtr + gamma.n_elem, size, 1, false, false);
+  beta = arma::mat(weights.memptr() + gamma.n_elem, size, 1, false, false);
 
   if (!loading)
   {
@@ -68,16 +69,18 @@ void BatchNorm<InputType, OutputType>::SetWeights(
     beta.fill(0.0);
   }
 
+  deterministic = false;
   loading = false;
 }
 
-template<typename InputType, typename OutputType>
-void BatchNorm<InputType, OutputType>::Forward(
-    const InputType& input,
-    OutputType& output)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void BatchNorm<InputDataType, OutputDataType>::Forward(
+    const arma::Mat<eT>& input,
+    arma::Mat<eT>& output)
 {
-  Log::Assert(input.n_rows % size == 0, "Input features must be divisible "
-      "by feature maps.");
+  Log::Assert(input.n_rows % size == 0, "Input features must be divisible \
+      by feature maps.");
 
   const size_t batchSize = input.n_cols;
   const size_t inputSize = input.n_rows / size;
@@ -86,7 +89,7 @@ void BatchNorm<InputType, OutputType>::Forward(
   output.set_size(arma::size(input));
 
   // We will calculate minibatch norm on each channel / feature map.
-  if (this->training)
+  if (!deterministic)
   {
     // Check only during training, batch-size can be one during inference.
     if (batchSize == 1 && inputSize == 1)
@@ -98,14 +101,12 @@ void BatchNorm<InputType, OutputType>::Forward(
 
     // Input corresponds to output from convolution layer.
     // Use a cube for simplicity.
-    arma::Cube<typename InputType::elem_type> inputTemp(
-        const_cast<InputType&>(input).memptr(), inputSize, size, batchSize,
-        false, false);
+    arma::cube inputTemp(const_cast<arma::Mat<eT>&>(input).memptr(),
+        inputSize, size, batchSize, false, false);
 
     // Initialize output to same size and values for convenience.
-    arma::Cube<typename OutputType::elem_type> outputTemp(
-        const_cast<OutputType&>(output).memptr(), inputSize, size, batchSize,
-        false, false);
+    arma::cube outputTemp(const_cast<arma::Mat<eT>&>(output).memptr(),
+        inputSize, size, batchSize, false, false);
     outputTemp = inputTemp;
 
     // Calculate mean and variance over all channels.
@@ -151,9 +152,8 @@ void BatchNorm<InputType, OutputType>::Forward(
   {
     // Normalize the input and scale and shift the output.
     output = input;
-    arma::Cube<typename OutputType::elem_type> outputTemp(
-        const_cast<OutputType&>(output).memptr(), input.n_rows / size, size,
-        batchSize, false, false);
+    arma::cube outputTemp(const_cast<arma::Mat<eT>&>(output).memptr(),
+        input.n_rows / size, size, batchSize, false, false);
 
     outputTemp.each_slice() -= arma::repmat(runningMean.t(),
         input.n_rows / size, 1);
@@ -166,29 +166,28 @@ void BatchNorm<InputType, OutputType>::Forward(
   }
 }
 
-template<typename InputType, typename OutputType>
-void BatchNorm<InputType, OutputType>::Backward(
-    const InputType& input,
-    const OutputType& gy,
-    OutputType& g)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void BatchNorm<InputDataType, OutputDataType>::Backward(
+    const arma::Mat<eT>& input,
+    const arma::Mat<eT>& gy,
+    arma::Mat<eT>& g)
 {
   const arma::mat stdInv = 1.0 / arma::sqrt(variance + eps);
 
   g.set_size(arma::size(input));
-  arma::Cube<typename OutputType::elem_type> gyTemp(
-      const_cast<OutputType&>(gy).memptr(), input.n_rows / size, size,
-      input.n_cols, false, false);
-  arma::Cube<typename OutputType::elem_type> gTemp(
-      const_cast<OutputType&>(g).memptr(), input.n_rows / size, size,
-      input.n_cols, false, false);
+  arma::cube gyTemp(const_cast<arma::Mat<eT>&>(gy).memptr(),
+      input.n_rows / size, size, input.n_cols, false, false);
+  arma::cube gTemp(const_cast<arma::Mat<eT>&>(g).memptr(),
+      input.n_rows / size, size, input.n_cols, false, false);
 
   // Step 1: dl / dxhat.
-  arma::Cube<typename OutputType::elem_type> norm =
-      gyTemp.each_slice() % arma::repmat(gamma.t(), input.n_rows / size, 1);
+  arma::cube norm = gyTemp.each_slice() % arma::repmat(gamma.t(),
+      input.n_rows / size, 1);
 
   // Step 2: sum dl / dxhat * (x - mu) * -0.5 * stdInv^3.
-  OutputType temp = arma::sum(norm % inputMean, 2);
-  OutputType vars = temp % arma::repmat(arma::pow(stdInv, 3),
+  arma::mat temp = arma::sum(norm % inputMean, 2);
+  arma::mat vars = temp % arma::repmat(arma::pow(stdInv, 3),
       input.n_rows / size, 1) * -0.5;
 
   // Step 3: dl / dxhat * 1 / stdInv + variance * 2 * (x - mu) / m +
@@ -199,25 +198,25 @@ void BatchNorm<InputType, OutputType>::Backward(
 
   // Step 4: sum (dl / dxhat * -1 / stdInv) + variance *
   // (sum -2 * (x - mu)) / m.
-  OutputType normTemp = arma::sum(norm.each_slice() %
+  arma::mat normTemp = arma::sum(norm.each_slice() %
       arma::repmat(-stdInv, input.n_rows / size, 1) , 2) /
       input.n_cols;
   gTemp.each_slice() += normTemp;
 }
 
-template<typename InputType, typename OutputType>
-void BatchNorm<InputType, OutputType>::Gradient(
-    const InputType& /* input */,
-    const OutputType& error,
-    OutputType& gradient)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void BatchNorm<InputDataType, OutputDataType>::Gradient(
+    const arma::Mat<eT>& /* input */,
+    const arma::Mat<eT>& error,
+    arma::Mat<eT>& gradient)
 {
   gradient.set_size(size + size, 1);
-  arma::Cube<typename OutputType::elem_type> errorTemp(
-      const_cast<OutputType&>(error).memptr(), error.n_rows / size, size,
-      error.n_cols, false, false);
+  arma::cube errorTemp(const_cast<arma::Mat<eT>&>(error).memptr(),
+      error.n_rows / size, size, error.n_cols, false, false);
 
   // Step 5: dl / dy * xhat.
-  OutputType temp = arma::sum(arma::sum(normalized % errorTemp, 0), 2);
+  arma::mat temp = arma::sum(arma::sum(normalized % errorTemp, 0), 2);
   gradient.submat(0, 0, gamma.n_elem - 1, 0) = temp.t();
 
   // Step 6: dl / dy.
@@ -225,27 +224,22 @@ void BatchNorm<InputType, OutputType>::Gradient(
   gradient.submat(gamma.n_elem, 0, gradient.n_elem - 1, 0) = temp.t();
 }
 
-template<typename InputType, typename OutputType>
+template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
-void BatchNorm<InputType, OutputType>::serialize(
+void BatchNorm<InputDataType, OutputDataType>::serialize(
     Archive& ar, const uint32_t /* version */)
 {
-  ar(cereal::base_class<Layer<InputType, OutputType>>(this));
-
   ar(CEREAL_NVP(size));
+
+  if (cereal::is_loading<Archive>())
+  {
+    weights.set_size(size + size, 1);
+    loading = true;
+  }
+
   ar(CEREAL_NVP(eps));
   ar(CEREAL_NVP(gamma));
   ar(CEREAL_NVP(beta));
-  ar(CEREAL_NVP(weights));
-
-  if (Archive::is_loading::value)
-  {
-    // Gamma acts as the scaling parameters for the normalized output.
-    gamma = arma::mat(weights.memptr(), size, 1, false, false);
-    // Beta acts as the shifting parameters for the normalized output.
-    beta = arma::mat(weights.memptr() + gamma.n_elem, size, 1, false, false);
-  }
-
   ar(CEREAL_NVP(count));
   ar(CEREAL_NVP(averageFactor));
   ar(CEREAL_NVP(momentum));

src/mlpack/methods/ann/layer/bilinear_interpolation.hpp

@@ -1,4 +1,3 @@
-// Temporarily drop.
 /**
  * @file methods/ann/layer/bilinear_interpolation.hpp
  * @author Kris Singh
@@ -28,30 +27,35 @@ namespace ann /** Artificial Neural Network. */ {
  * different known points in the grid. This way, we represent any arbitrary
  * point, present within the grid, as a function of those four points.
  *
- * @tparam InputType Type of the input data (arma::colvec, arma::mat,
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
- * @tparam OutputType Type of the output data (arma::colvec, arma::mat,
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
  */
 template <
-    typename InputType = arma::mat,
-    typename OutputType = arma::mat
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
 >
-class BilinearInterpolationType : public Layer<InputType, OutputType>
+class BilinearInterpolation
 {
  public:
-  //! Create the BilinearInterpolationType object.
-  BilinearInterpolationType();
+  //! Create the Bilinear Interpolation object.
+  BilinearInterpolation();
 
   /**
-   * The constructor for the Bilinear Interpolation.  The input size will be set
-   * by the given input when the layer is used.
+   * The constructor for the Bilinear Interpolation.
    *
+   * @param inRowSize Number of input rows.
+   * @param inColSize Number of input columns.
    * @param outRowSize Number of output rows.
    * @param outColSize Number of output columns.
+   * @param depth Number of input slices.
    */
-  BilinearInterpolationType(const size_t outRowSize,
-                            const size_t outColSize);
+  BilinearInterpolation(const size_t inRowSize,
+                        const size_t inColSize,
+                        const size_t outRowSize,
+                        const size_t outColSize,
+                        const size_t depth);
 
   /**
    * Forward pass through the layer. The layer interpolates
@@ -60,7 +64,8 @@ class BilinearInterpolationType : public Layer<InputType, OutputType>
    * @param input The input matrix.
    * @param output The resulting interpolated output matrix.
    */
-  void Forward(const InputType& input, OutputType& output);
+  template<typename eT>
+  void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);
 
   /**
    * Ordinary feed backward pass of a neural network, calculating the function
@@ -73,22 +78,30 @@ class BilinearInterpolationType : public Layer<InputType, OutputType>
    * @param gradient The computed backward gradient.
    * @param output The resulting down-sampled output.
    */
-  void Backward(const InputType& /*input*/,
-                const OutputType& gradient,
-                OutputType& output);
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& /*input*/,
+                const arma::Mat<eT>& gradient,
+                arma::Mat<eT>& output);
 
-  const std::vector<size_t>& OutputDimensions() const
-  {
-    std::vector<size_t> result(this->inputDimensions.size(), 0);
-    result[0] = outRowSize;
-    result[1] = outColSize;
-    if (result.size() > 2)
-    {
-      for (size_t i = 0; i < result.size(); ++i)
-        result[i] = this->inputDimensions[i];
-    }
-    return result;
-  }
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  //! Get the row size of the input.
+  size_t const& InRowSize() const { return inRowSize; }
+  //! Modify the row size of the input.
+  size_t& InRowSize() { return inRowSize; }
+
+  //! Get the column size of the input.
+  size_t const& InColSize() const { return inColSize; }
+  //! Modify the column size of the input.
+  size_t& InColSize() { return inColSize; }
 
   //! Get the row size of the output.
   size_t const& OutRowSize() const { return outRowSize; }
@@ -100,6 +113,17 @@ class BilinearInterpolationType : public Layer<InputType, OutputType>
   //! Modify the column size of the output.
   size_t& OutColSize() { return outColSize; }
 
+  //! Get the depth of the input.
+  size_t const& InDepth() const { return depth; }
+  //! Modify the depth of the input.
+  size_t& InDepth() { return depth; }
+
+  //! Get the shape of the input.
+  size_t InputShape() const
+  {
+    return inRowSize;
+  }
+
   /**
    * Serialize the layer.
    */
@@ -107,16 +131,24 @@ class BilinearInterpolationType : public Layer<InputType, OutputType>
   void serialize(Archive& ar, const uint32_t /* version */);
 
  private:
+  //! Locally stored row size of the input.
+  size_t inRowSize;
+  //! Locally stored column size of the input.
+  size_t inColSize;
   //! Locally stored row size of the output.
   size_t outRowSize;
-
   //! Locally stored column size of the input.
   size_t outColSize;
+  //! Locally stored depth of the input.
+  size_t depth;
+  //! Locally stored number of input points.
+  size_t batchSize;
+  //! Locally-stored delta object.
+  OutputDataType delta;
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
 }; // class BilinearInterpolation
 
-// Standard BilinearInterpolation layer.
-typedef BilinearInterpolationType<arma::mat, arma::mat> BilinearInterpolation;
-
 } // namespace ann
 } // namespace mlpack
 

src/mlpack/methods/ann/layer/bilinear_interpolation_impl.hpp

@@ -19,54 +19,69 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template<typename InputType, typename OutputType>
-BilinearInterpolationType<InputType, OutputType>::
-BilinearInterpolationType():
+
+template<typename InputDataType, typename OutputDataType>
+BilinearInterpolation<InputDataType, OutputDataType>::
+BilinearInterpolation():
+    inRowSize(0),
+    inColSize(0),
     outRowSize(0),
-    outColSize(0)
+    outColSize(0),
+    depth(0),
+    batchSize(0)
 {
   // Nothing to do here.
 }
 
-template<typename InputType, typename OutputType>
-BilinearInterpolationType<InputType, OutputType>::
-BilinearInterpolationType(const size_t outRowSize,
-                          const size_t outColSize) :
+template<typename InputDataType, typename OutputDataType>
+BilinearInterpolation<InputDataType, OutputDataType>::
+BilinearInterpolation(
+    const size_t inRowSize,
+    const size_t inColSize,
+    const size_t outRowSize,
+    const size_t outColSize,
+    const size_t depth):
+    inRowSize(inRowSize),
+    inColSize(inColSize),
     outRowSize(outRowSize),
-    outColSize(outColSize)
+    outColSize(outColSize),
+    depth(depth),
+    batchSize(0)
 {
   // Nothing to do here.
 }
 
-template<typename InputType, typename OutputType>
-void BilinearInterpolationType<InputType, OutputType>::Forward(
-    const InputType& input, OutputType& output)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void BilinearInterpolation<InputDataType, OutputDataType>::Forward(
+    const arma::Mat<eT>& input, arma::Mat<eT>& output)
 {
-  const size_t batchSize = input.n_cols;
-  const size_t depth = this->inputDimensions.size() <= 2 ? 1 :
-      std::accumulate(this->inputDimensions.begin() + 2, this->inputDimensions.end(), 0);
+  batchSize = input.n_cols;
+  if (output.is_empty())
+    output.set_size(outRowSize * outColSize * depth, batchSize);
+  else
+  {
+    assert(output.n_rows == outRowSize * outColSize * depth);
+    assert(output.n_cols == batchSize);
+  }
 
-  assert(output.n_rows == outRowSize * outColSize * depth);
-  assert(output.n_cols == batchSize);
+  assert(inRowSize >= 2);
+  assert(inColSize >= 2);
 
-  assert(this->inputDimensions[0] >= 2);
-  assert(this->inputDimensions[1] >= 2);
+  arma::cube inputAsCube(const_cast<arma::Mat<eT>&>(input).memptr(),
+      inRowSize, inColSize, depth * batchSize, false, false);
+  arma::cube outputAsCube(output.memptr(), outRowSize, outColSize,
+                          depth * batchSize, false, true);
 
-  arma::Cube<typename InputType::elem_type> inputAsCube(
-      const_cast<InputType&>(input).memptr(), this->inputDimensions[0],
-      this->inputDimensions[1], depth * batchSize, false, false);
-  arma::Cube<typename OutputType::elem_type> outputAsCube(
-      output.memptr(), outRowSize, outColSize, depth * batchSize, false, true);
-
-  double scaleRow = (double) this->inputDimensions[0] / (double) outRowSize;
-  double scaleCol = (double) this->inputDimensions[1] / (double) outColSize;
+  double scaleRow = (double) inRowSize / (double) outRowSize;
+  double scaleCol = (double) inColSize / (double) outColSize;
 
   arma::mat22 coeffs;
   for (size_t i = 0; i < outRowSize; ++i)
   {
     size_t rOrigin = (size_t) std::floor(i * scaleRow);
-    if (rOrigin > this->inputDimensions[0] - 2)
-      rOrigin = this->inputDimensions[0] - 2;
+    if (rOrigin > inRowSize - 2)
+      rOrigin = inRowSize - 2;
 
     // Scaled distance of the interpolated point from the topmost row.
     double deltaR = i * scaleRow - rOrigin;
@@ -76,8 +91,8 @@ void BilinearInterpolationType<InputType, OutputType>::Forward(
     {
       // Scaled distance of the interpolated point from the leftmost column.
       size_t cOrigin = (size_t) std::floor(j * scaleCol);
-      if (cOrigin > this->inputDimensions[1] - 2)
-        cOrigin = this->inputDimensions[1] - 2;
+      if (cOrigin > inColSize - 2)
+        cOrigin = inColSize - 2;
 
       double deltaC = j * scaleCol - cOrigin;
       if (deltaC > 1)
@@ -96,27 +111,28 @@ void BilinearInterpolationType<InputType, OutputType>::Forward(
   }
 }
 
-template<typename InputType, typename OutputType>
-void BilinearInterpolationType<InputType, OutputType>::Backward(
-    const InputType& /*input*/,
-    const OutputType& gradient,
-    OutputType& output)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void BilinearInterpolation<InputDataType, OutputDataType>::Backward(
+    const arma::Mat<eT>& /*input*/,
+    const arma::Mat<eT>& gradient,
+    arma::Mat<eT>& output)
 {
-  const size_t batchSize = output.n_cols;
-  const size_t depth = this->inputDimensions.size() <= 2 ? 1 :
-      std::accumulate(this->inputDimensions.begin() + 2, this->inputDimensions.end(), 0);
-
-  assert(output.n_rows == this->inputDimensions[0] * this->inputDimensions[1] * depth);
+  if (output.is_empty())
+    output.set_size(inRowSize * inColSize * depth, batchSize);
+  else
+  {
+    assert(output.n_rows == inRowSize * inColSize * depth);
+    assert(output.n_cols == batchSize);
+  }
 
   assert(outRowSize >= 2);
   assert(outColSize >= 2);
 
-  arma::Cube<typename OutputType::elem_type> gradientAsCube(
-      ((OutputType&) gradient).memptr(), outRowSize, outColSize, depth *
-      batchSize, false, false);
-  arma::Cube<typename OutputType::elem_type> outputAsCube(
-      output.memptr(), this->inputDimensions[0], this->inputDimensions[1], depth * batchSize,
-      false, true);
+  arma::cube gradientAsCube(((arma::Mat<eT>&) gradient).memptr(), outRowSize,
+      outColSize, depth * batchSize, false, false);
+  arma::cube outputAsCube(output.memptr(), inRowSize, inColSize,
+                          depth * batchSize, false, true);
 
   if (gradient.n_elem == output.n_elem)
   {
@@ -124,17 +140,17 @@ void BilinearInterpolationType<InputType, OutputType>::Backward(
   }
   else
   {
-    double scaleRow = (double)(outRowSize) / this->inputDimensions[0];
-    double scaleCol = (double)(outColSize) / this->inputDimensions[1];
+    double scaleRow = (double)(outRowSize) / inRowSize;
+    double scaleCol = (double)(outColSize) / inColSize;
 
     arma::mat22 coeffs;
-    for (size_t i = 0; i < this->inputDimensions[0]; ++i)
+    for (size_t i = 0; i < inRowSize; ++i)
     {
       size_t rOrigin = (size_t) std::floor(i * scaleRow);
       if (rOrigin > outRowSize - 2)
         rOrigin = outRowSize - 2;
       double deltaR = i * scaleRow - rOrigin;
-      for (size_t j = 0; j < this->inputDimensions[1]; ++j)
+      for (size_t j = 0; j < inColSize; ++j)
       {
         size_t cOrigin = (size_t) std::floor(j * scaleCol);
 
@@ -157,15 +173,16 @@ void BilinearInterpolationType<InputType, OutputType>::Backward(
   }
 }
 
-template<typename InputType, typename OutputType>
+template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
-void BilinearInterpolationType<InputType, OutputType>::serialize(
+void BilinearInterpolation<InputDataType, OutputDataType>::serialize(
     Archive& ar, const uint32_t /* version */)
 {
-  ar(cereal::base_class<Layer<InputType, OutputType>>(this));
-
+  ar(CEREAL_NVP(inRowSize));
+  ar(CEREAL_NVP(inColSize));
   ar(CEREAL_NVP(outRowSize));
   ar(CEREAL_NVP(outColSize));
+  ar(CEREAL_NVP(depth));
 }
 
 } // namespace ann

src/mlpack/methods/ann/layer/c_relu.hpp

@@ -14,12 +14,10 @@
 
 #include <mlpack/prereqs.hpp>
 
-#include "layer.hpp"
-
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
-
 /**
+ *
  * A concatenated ReLU has two outputs, one ReLU and one negative ReLU,
  * concatenated together. In other words, for positive x it produces [x, 0],
  * and for negative x it produces [0, x]. Because it has two outputs,
@@ -40,21 +38,22 @@ namespace ann /** Artificial Neural Network. */ {
  * }
  * @endcode
  *
- * @tparam InputType The type of the layer's inputs. The layer automatically
- *     cast inputs to this type (Default: arma::mat).
- * @tparam OutputType The type of the computation which also causes the output
- *     to also be in this type. The type also allows the computation and weight
- *     type to differ from the input type (Default: arma::mat).
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
  */
-template<typename InputType = arma::mat, typename OutputType = arma::mat>
-class CReLUType : public Layer<InputType, OutputType>
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+class CReLU
 {
  public:
-  //! Create the CReLU object.
-  CReLUType();
-
-  //! Clone the CReLUType object. This handles polymorphism correctly.
-  CReLUType* Clone() const { return new CReLUType(*this); }
+  /**
+   * Create the CReLU object.
+   */
+  CReLU();
 
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function
@@ -64,6 +63,7 @@ class CReLUType : public Layer<InputType, OutputType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
+  template<typename InputType, typename OutputType>
   void Forward(const InputType& input, OutputType& output);
 
   /**
@@ -75,17 +75,35 @@ class CReLUType : public Layer<InputType, OutputType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const InputType& input, const OutputType& gy, OutputType& g);
+  template<typename DataType>
+  void Backward(const DataType& input, const DataType& gy, DataType& g);
 
-  //! Serialize the layer.
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  //! Get size of weights.
+  size_t WeightSize() const { return 0; }
+
+  /**
+   * Serialize the layer.
+   */
   template<typename Archive>
   void serialize(Archive& /* ar */, const uint32_t /* version */);
-}; // class CReLUType
 
-// Convenience typedefs.
+ private:
+  //! Locally-stored delta object.
+  OutputDataType delta;
 
-// Standard CReLU layer.
-typedef CReLUType<arma::mat, arma::mat> CReLU;
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+}; // class CReLU
 
 } // namespace ann
 } // namespace mlpack

src/mlpack/methods/ann/layer/c_relu_impl.hpp

@@ -18,36 +18,39 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template<typename InputType, typename OutputType>
-CReLUType<InputType, OutputType>::CReLUType()
+template<typename InputDataType, typename OutputDataType>
+CReLU<InputDataType, OutputDataType>::CReLU()
 {
   // Nothing to do here.
 }
 
+template<typename InputDataType, typename OutputDataType>
 template<typename InputType, typename OutputType>
-void CReLUType<InputType, OutputType>::Forward(
+void CReLU<InputDataType, OutputDataType>::Forward(
     const InputType& input, OutputType& output)
 {
   output = arma::join_cols(arma::max(input, 0.0 * input), arma::max(
       (-1 * input), 0.0 * input));
 }
 
-template<typename InputType, typename OutputType>
-void CReLUType<InputType, OutputType>::Backward(
-    const InputType& input, const OutputType& gy, OutputType& g)
+template<typename InputDataType, typename OutputDataType>
+template<typename DataType>
+void CReLU<InputDataType, OutputDataType>::Backward(
+    const DataType& input, const DataType& gy, DataType& g)
 {
-  OutputType temp = gy % (input >= 0.0);
+  DataType temp;
+  temp = gy % (input >= 0.0);
   g = temp.rows(0, (input.n_rows / 2 - 1)) - temp.rows(input.n_rows / 2,
       (input.n_rows - 1));
 }
 
-template<typename InputType, typename OutputType>
+template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
-void CReLUType<InputType, OutputType>::serialize(
-    Archive& ar,
+void CReLU<InputDataType, OutputDataType>::serialize(
+    Archive& /* ar */,
     const uint32_t /* version */)
 {
-  ar(cereal::base_class<Layer<InputType, OutputType>>(this));
+  // Nothing to do here.
 }
 
 } // namespace ann

src/mlpack/methods/ann/layer/celu.hpp

@@ -25,8 +25,6 @@
 
 #include <mlpack/prereqs.hpp>
 
-#include "layer.hpp"
-
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
@@ -48,16 +46,18 @@ namespace ann /** Artificial Neural Network. */ {
  * \right.
  * @f}
  *
- * When not in training mode, there is no computation of the derivative.
+ * In the deterministic mode, there is no computation of the derivative.
  *
- * @tparam InputType The type of the layer's inputs. The layer automatically
- *     cast inputs to this type (Default: arma::mat).
- * @tparam OutputType The type of the computation which also causes the output
- *     to also be in this type. The type also allows the computation and weight
- *     type to differ from the input type (Default: arma::mat).
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
  */
-template<typename InputType = arma::mat, typename OutputType = arma::mat>
-class CELUType : public Layer<InputType, OutputType>
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+class CELU
 {
  public:
   /**
@@ -67,10 +67,7 @@ class CELUType : public Layer<InputType, OutputType>
    *
    * @param alpha Scale parameter for the negative factor (default = 1.0).
    */
-  CELUType(const double alpha = 1.0);
-
-  //! Clone the CELUType object. This handles polymorphism correctly.
-  CELUType* Clone() const { return new CELUType(*this); }
+  CELU(const double alpha = 1.0);
 
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function
@@ -79,6 +76,7 @@ class CELUType : public Layer<InputType, OutputType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
+  template<typename InputType, typename OutputType>
   void Forward(const InputType& input, OutputType& output);
 
   /**
@@ -90,29 +88,54 @@ class CELUType : public Layer<InputType, OutputType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const InputType& input, const OutputType& gy, OutputType& g);
+  template<typename DataType>
+  void Backward(const DataType& input, const DataType& gy, DataType& g);
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
 
   //! Get the non zero gradient.
   double const& Alpha() const { return alpha; }
   //! Modify the non zero gradient.
   double& Alpha() { return alpha; }
 
-  //! Serialize the layer.
+  //! Get the value of deterministic parameter.
+  bool Deterministic() const { return deterministic; }
+  //! Modify the value of deterministic parameter.
+  bool& Deterministic() { return deterministic; }
+
+  //! Get size of weights.
+  size_t WeightSize() { return 0; }
+
+  /**
+   * Serialize the layer.
+   */
   template<typename Archive>
   void serialize(Archive& ar, const uint32_t /* version */);
 
  private:
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
   //! Locally stored first derivative of the activation function.
-  OutputType derivative;
+  arma::mat derivative;
 
   //! CELU Hyperparameter (alpha > 0).
   double alpha;
-}; // class CELUType
 
-// Convenience typedefs.
-
-// Standard CELU layer.
-typedef CELUType<arma::mat, arma::mat> CELU;
+  //! If true the derivative computation is disabled, see notes above.
+  bool deterministic;
+}; // class CELU
 
 } // namespace ann
 } // namespace mlpack

src/mlpack/methods/ann/layer/celu_impl.hpp

@@ -18,9 +18,10 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template<typename InputType, typename OutputType>
-CELUType<InputType, OutputType>::CELUType(const double alpha) :
-    alpha(alpha)
+template<typename InputDataType, typename OutputDataType>
+CELU<InputDataType, OutputDataType>::CELU(const double alpha) :
+    alpha(alpha),
+    deterministic(false)
 {
   if (alpha == 0)
   {
@@ -29,18 +30,19 @@ CELUType<InputType, OutputType>::CELUType(const double alpha) :
   }
 }
 
+template<typename InputDataType, typename OutputDataType>
 template<typename InputType, typename OutputType>
-void CELUType<InputType, OutputType>::Forward(
+void CELU<InputDataType, OutputDataType>::Forward(
     const InputType& input, OutputType& output)
 {
-  output = arma::ones<OutputType>(arma::size(input));
+  output = arma::ones<OutputDataType>(arma::size(input));
   for (size_t i = 0; i < input.n_elem; ++i)
   {
     output(i) = (input(i) >= 0) ? input(i) : alpha *
-        (std::exp(input(i) / alpha) - 1);
+                (std::exp(input(i) / alpha) - 1);
   }
 
-  if (this->training)
+  if (!deterministic)
   {
     derivative.set_size(arma::size(input));
     for (size_t i = 0; i < input.n_elem; ++i)
@@ -51,24 +53,21 @@ void CELUType<InputType, OutputType>::Forward(
   }
 }
 
-template<typename InputType, typename OutputType>
-void CELUType<InputType, OutputType>::Backward(
-    const InputType& /* input */, const OutputType& gy, OutputType& g)
+template<typename InputDataType, typename OutputDataType>
+template<typename DataType>
+void CELU<InputDataType, OutputDataType>::Backward(
+    const DataType& /* input */, const DataType& gy, DataType& g)
 {
   g = gy % derivative;
 }
 
-template<typename InputType, typename OutputType>
+template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
-void CELUType<InputType, OutputType>::serialize(
+void CELU<InputDataType, OutputDataType>::serialize(
     Archive& ar,
     const uint32_t /* version */)
 {
-  ar(cereal::base_class<Layer<InputType, OutputType>>(this));
-
   ar(CEREAL_NVP(alpha));
-  if (Archive::is_loading::value)
-    derivative.clear();
 }
 
 } // namespace ann

src/mlpack/methods/ann/layer/concat.hpp

@@ -0,0 +1,263 @@
+/**
+ * @file methods/ann/layer/concat.hpp
+ * @author Marcus Edel
+ * @author Mehul Kumar Nirala
+ *
+ * Definition of the Concat class, which acts as a concatenation container.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_CONCAT_HPP
+#define MLPACK_METHODS_ANN_LAYER_CONCAT_HPP
+
+#include <mlpack/prereqs.hpp>
+
+#include "../visitor/delete_visitor.hpp"
+#include "../visitor/delta_visitor.hpp"
+#include "../visitor/output_parameter_visitor.hpp"
+
+#include "layer_types.hpp"
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * Implementation of the Concat class. The Concat class works as a
+ * feed-forward fully connected network container which plugs various layers
+ * together.
+ *
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam CustomLayers Additional custom layers if required.
+ */
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat,
+    typename... CustomLayers
+>
+class Concat
+{
+ public:
+  /**
+   * Create the Concat object using the specified parameters.
+   *
+   * @param model Expose all network modules.
+   * @param run Call the Forward/Backward method before the output is merged.
+   */
+  Concat(const bool model = false,
+         const bool run = true);
+
+  /**
+   * Create the Concat object using the specified parameters.
+   *
+   * @param inputSize A vector denoting input size of each layer added.
+   * @param axis Concat axis.
+   * @param model Expose all network modules.
+   * @param run Call the Forward/Backward method before the output is merged.
+   */
+  Concat(arma::Row<size_t>& inputSize,
+         const size_t axis,
+         const bool model = false,
+         const bool run = true);
+
+  /**
+   * Destroy the layers held by the model.
+   */
+  ~Concat();
+
+  /**
+   * Ordinary feed forward pass of a neural network, evaluating the function
+   * f(x) by propagating the activity forward through f.
+   *
+   * @param input Input data used for evaluating the specified function.
+   * @param output Resulting output activation.
+   */
+  template<typename eT>
+  void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);
+
+  /**
+   * Ordinary feed backward pass of a neural network, using 3rd-order tensors as
+   * input, calculating the function f(x) by propagating x backwards through f.
+   * Using the results from the feed forward pass.
+   *
+   * @param * (input) The propagated input activation.
+   * @param gy The backpropagated error.
+   * @param g The calculated gradient.
+   */
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& /* input */,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g);
+
+  /**
+   * This is the overload of Backward() that runs only a specific layer with
+   * the given input.
+   *
+   * @param * (input) The propagated input activation.
+   * @param gy The backpropagated error.
+   * @param g The calculated gradient.
+   * @param index The index of the layer to run.
+   */
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& /* input */,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g,
+                const size_t index);
+
+  /*
+   * Calculate the gradient using the output delta and the input activation.
+   *
+   * @param input The input parameter used for calculating the gradient.
+   * @param error The calculated error.
+   * @param gradient The calculated gradient.
+   */
+  template<typename eT>
+  void Gradient(const arma::Mat<eT>& /* input */,
+                const arma::Mat<eT>& error,
+                arma::Mat<eT>& /* gradient */);
+
+  /*
+   * This is the overload of Gradient() that runs a specific layer with the
+   * given input.
+   *
+   * @param input The input parameter used for calculating the gradient.
+   * @param error The calculated error.
+   * @param gradient The calculated gradient.
+   * @param The index of the layer to run.
+   */
+  template<typename eT>
+  void Gradient(const arma::Mat<eT>& input,
+                const arma::Mat<eT>& error,
+                arma::Mat<eT>& gradient,
+                const size_t index);
+
+  /*
+   * Add a new module to the model.
+   *
+   * @param args The layer parameter.
+   */
+  template <class LayerType, class... Args>
+  void Add(Args... args) { network.push_back(new LayerType(args...)); }
+
+  /*
+   * Add a new module to the model.
+   *
+   * @param layer The Layer to be added to the model.
+   */
+  void Add(LayerTypes<CustomLayers...> layer) { network.push_back(layer); }
+
+  //! Return the model modules.
+  std::vector<LayerTypes<CustomLayers...> >& Model()
+  {
+    if (model)
+    {
+      return network;
+    }
+
+    return empty;
+  }
+
+  //! Return the initial point for the optimization.
+  const arma::mat& Parameters() const { return weights; }
+  //! Modify the initial point for the optimization.
+  arma::mat& Parameters() { return weights; }
+
+  //! Get the value of run parameter.
+  bool Run() const { return run; }
+  //! Modify the value of run parameter.
+  bool& Run() { return run; }
+
+  arma::mat const& InputParameter() const { return inputParameter; }
+  //! Modify the input parameter.
+  arma::mat& InputParameter() { return inputParameter; }
+
+  //! Get the output parameter.
+  arma::mat const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  arma::mat& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.e
+  arma::mat const& Delta() const { return delta; }
+  //! Modify the delta.
+  arma::mat& Delta() { return delta; }
+
+  //! Get the gradient.
+  arma::mat const& Gradient() const { return gradient; }
+  //! Modify the gradient.
+  arma::mat& Gradient() { return gradient; }
+
+  //! Get the axis of concatenation.
+  size_t const& ConcatAxis() const { return axis; }
+
+  //! Get the size of the weight matrix.
+  size_t WeightSize() const { return 0; }
+
+  /**
+   * Serialize the layer
+   */
+  template<typename Archive>
+  void serialize(Archive& ar,  const uint32_t /* version */);
+
+ private:
+  //! Parameter which indicates the input size of modules.
+  arma::Row<size_t> inputSize;
+
+  //! Parameter which indicates the axis of concatenation.
+  size_t axis;
+
+  //! Parameter which indicates whether to use the axis of concatenation.
+  bool useAxis;
+
+  //! Parameter which indicates if the modules should be exposed.
+  bool model;
+
+  //! Parameter which indicates if the Forward/Backward method should be called
+  //! before merging the output.
+  bool run;
+
+  //! Parameter to store channels.
+  size_t channels;
+
+  //! Locally-stored network modules.
+  std::vector<LayerTypes<CustomLayers...> > network;
+
+  //! Locally-stored model weights.
+  OutputDataType weights;
+
+  //! Locally-stored delta visitor.
+  DeltaVisitor deltaVisitor;
+
+  //! Locally-stored output parameter visitor.
+  OutputParameterVisitor outputParameterVisitor;
+
+  //! Locally-stored delete visitor.
+  DeleteVisitor deleteVisitor;
+
+  //! Locally-stored empty list of modules.
+  std::vector<LayerTypes<CustomLayers...> > empty;
+
+  //! Locally-stored delta object.
+  arma::mat delta;
+
+  //! Locally-stored input parameter object.
+  arma::mat inputParameter;
+
+  //! Locally-stored output parameter object.
+  arma::mat outputParameter;
+
+  //! Locally-stored gradient object.
+  arma::mat gradient;
+}; // class Concat
+
+} // namespace ann
+} // namespace mlpack
+
+// Include implementation.
+#include "concat_impl.hpp"
+
+#endif

src/mlpack/methods/ann/layer/concat_impl.hpp

@@ -0,0 +1,293 @@
+/**
+ * @file methods/ann/layer/concat_impl.hpp
+ * @author Marcus Edel
+ * @author Mehul Kumar Nirala
+ *
+ * Implementation of the Concat class, which acts as a concatenation contain.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_CONCAT_IMPL_HPP
+#define MLPACK_METHODS_ANN_LAYER_CONCAT_IMPL_HPP
+
+// In case it hasn't yet been included.
+#include "concat.hpp"
+
+#include "../visitor/forward_visitor.hpp"
+#include "../visitor/backward_visitor.hpp"
+#include "../visitor/gradient_visitor.hpp"
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+Concat<InputDataType, OutputDataType, CustomLayers...>::Concat(
+    const bool model, const bool run) :
+    axis(0),
+    useAxis(false),
+    model(model),
+    run(run),
+    channels(1)
+{
+  weights.set_size(0, 0);
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+Concat<InputDataType, OutputDataType, CustomLayers...>::Concat(
+    arma::Row<size_t>& inputSize,
+    const size_t axis,
+    const bool model,
+    const bool run) :
+    inputSize(inputSize),
+    axis(axis),
+    useAxis(true),
+    model(model),
+    run(run)
+{
+  weights.set_size(0, 0);
+
+  // Parameters to help calculate the number of channels.
+  size_t oldColSize = 1, newColSize = 1;
+  // Axis is specified and useAxis is true.
+  if (useAxis)
+  {
+    // Axis is specified without input dimension.
+    // Throw an error.
+    if (inputSize.n_elem > 0)
+    {
+      // Calculate rowSize, newColSize based on the axis
+      // of concatenation. Finally concat along cols and
+      // reshape to original format i.e. (input, batch_size).
+      size_t i = std::min(axis + 1, (size_t) inputSize.n_elem);
+      for (; i < inputSize.n_elem; ++i)
+      {
+        newColSize *= inputSize[i];
+      }
+    }
+    else
+    {
+      throw std::logic_error("Input dimensions not specified.");
+    }
+  }
+  else
+  {
+    channels = 1;
+  }
+  if (newColSize <= 0)
+  {
+      throw std::logic_error("Col size is zero.");
+  }
+  channels = newColSize / oldColSize;
+  inputSize.clear();
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+Concat<InputDataType, OutputDataType, CustomLayers...>::~Concat()
+{
+  if (!model)
+  {
+    // Clear memory.
+    std::for_each(network.begin(), network.end(),
+        boost::apply_visitor(deleteVisitor));
+  }
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+template<typename eT>
+void Concat<InputDataType, OutputDataType, CustomLayers...>::Forward(
+    const arma::Mat<eT>& input, arma::Mat<eT>& output)
+{
+  if (run)
+  {
+    for (size_t i = 0; i < network.size(); ++i)
+    {
+      boost::apply_visitor(ForwardVisitor(input,
+          boost::apply_visitor(outputParameterVisitor, network[i])),
+          network[i]);
+    }
+  }
+
+  output = boost::apply_visitor(outputParameterVisitor, network.front());
+
+  // Reshape output to incorporate the channels.
+  output.reshape(output.n_rows / channels, output.n_cols * channels);
+
+  for (size_t i = 1; i < network.size(); ++i)
+  {
+    arma::Mat<eT> out = boost::apply_visitor(outputParameterVisitor,
+        network[i]);
+
+    out.reshape(out.n_rows / channels, out.n_cols * channels);
+
+    // Vertically concatentate output from each layer.
+    output = arma::join_cols(output, out);
+  }
+  // Reshape output to its original shape.
+  output.reshape(output.n_rows * channels, output.n_cols / channels);
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+template<typename eT>
+void Concat<InputDataType, OutputDataType, CustomLayers...>::Backward(
+    const arma::Mat<eT>& /* input */, const arma::Mat<eT>& gy, arma::Mat<eT>& g)
+{
+  size_t rowCount = 0;
+  if (run)
+  {
+    arma::Mat<eT> delta;
+    arma::Mat<eT> gyTmp(((arma::Mat<eT>&) gy).memptr(), gy.n_rows / channels,
+        gy.n_cols * channels, false, false);
+    for (size_t i = 0; i < network.size(); ++i)
+    {
+      // Use rows from the error corresponding to the output from each layer.
+      size_t rows = boost::apply_visitor(
+          outputParameterVisitor, network[i]).n_rows;
+
+      // Extract from gy the parameters for the i-th network.
+      delta = gyTmp.rows(rowCount / channels, (rowCount + rows) / channels - 1);
+      delta.reshape(delta.n_rows * channels, delta.n_cols / channels);
+
+      boost::apply_visitor(BackwardVisitor(
+          boost::apply_visitor(outputParameterVisitor,
+          network[i]), delta,
+          boost::apply_visitor(deltaVisitor, network[i])), network[i]);
+      rowCount += rows;
+    }
+
+    g = boost::apply_visitor(deltaVisitor, network[0]);
+    for (size_t i = 1; i < network.size(); ++i)
+    {
+      g += boost::apply_visitor(deltaVisitor, network[i]);
+    }
+  }
+  else
+  {
+    g = gy;
+  }
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+template<typename eT>
+void Concat<InputDataType, OutputDataType, CustomLayers...>::Backward(
+    const arma::Mat<eT>& /* input */,
+    const arma::Mat<eT>& gy,
+    arma::Mat<eT>& g,
+    const size_t index)
+{
+  size_t rowCount = 0, rows = 0;
+
+  for (size_t i = 0; i < index; ++i)
+  {
+    rowCount += boost::apply_visitor(
+        outputParameterVisitor, network[i]).n_rows;
+  }
+  rows = boost::apply_visitor(outputParameterVisitor, network[index]).n_rows;
+
+  // Reshape gy to extract the i-th layer gy.
+  arma::Mat<eT> gyTmp(((arma::Mat<eT>&) gy).memptr(), gy.n_rows / channels,
+      gy.n_cols * channels, false, false);
+
+  arma::Mat<eT> delta = gyTmp.rows(rowCount / channels, (rowCount + rows) /
+      channels - 1);
+  delta.reshape(delta.n_rows * channels, delta.n_cols / channels);
+
+  boost::apply_visitor(BackwardVisitor(boost::apply_visitor(
+      outputParameterVisitor, network[index]), delta,
+      boost::apply_visitor(deltaVisitor, network[index])), network[index]);
+
+  g = boost::apply_visitor(deltaVisitor, network[index]);
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+template<typename eT>
+void Concat<InputDataType, OutputDataType, CustomLayers...>::Gradient(
+    const arma::Mat<eT>& input,
+    const arma::Mat<eT>& error,
+    arma::Mat<eT>& /* gradient */)
+{
+  if (run)
+  {
+    size_t rowCount = 0;
+    // Reshape error to extract the i-th layer error.
+    arma::Mat<eT> errorTmp(((arma::Mat<eT>&) error).memptr(),
+        error.n_rows / channels, error.n_cols * channels, false, false);
+    for (size_t i = 0; i < network.size(); ++i)
+    {
+      size_t rows = boost::apply_visitor(
+          outputParameterVisitor, network[i]).n_rows;
+
+      // Extract from error the parameters for the i-th network.
+      arma::Mat<eT> err = errorTmp.rows(rowCount / channels, (rowCount + rows) /
+          channels - 1);
+      err.reshape(err.n_rows * channels, err.n_cols / channels);
+
+      boost::apply_visitor(GradientVisitor(input, err), network[i]);
+      rowCount += rows;
+    }
+  }
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+template<typename eT>
+void Concat<InputDataType, OutputDataType, CustomLayers...>::Gradient(
+    const arma::Mat<eT>& input,
+    const arma::Mat<eT>& error,
+    arma::Mat<eT>& /* gradient */,
+    const size_t index)
+{
+  size_t rowCount = 0;
+  for (size_t i = 0; i < index; ++i)
+  {
+    rowCount += boost::apply_visitor(outputParameterVisitor,
+        network[i]).n_rows;
+  }
+  size_t rows = boost::apply_visitor(
+      outputParameterVisitor, network[index]).n_rows;
+
+  arma::Mat<eT> errorTmp(((arma::Mat<eT>&) error).memptr(),
+      error.n_rows / channels, error.n_cols * channels, false, false);
+  arma::Mat<eT> err = errorTmp.rows(rowCount / channels, (rowCount + rows) /
+      channels - 1);
+  err.reshape(err.n_rows * channels, err.n_cols / channels);
+
+  boost::apply_visitor(GradientVisitor(input, err), network[index]);
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+template<typename Archive>
+void Concat<InputDataType, OutputDataType, CustomLayers...>::serialize(
+    Archive& ar, const uint32_t /* version */)
+{
+  ar(CEREAL_NVP(model));
+  ar(CEREAL_NVP(run));
+
+  // Do we have to load or save a model?
+  if (model)
+  {
+    // Clear memory first, if needed.
+    if (cereal::is_loading<Archive>())
+    {
+      std::for_each(network.begin(), network.end(),
+          boost::apply_visitor(deleteVisitor));
+    }
+    ar(CEREAL_VECTOR_VARIANT_POINTER(network));
+  }
+}
+
+} // namespace ann
+} // namespace mlpack
+
+
+#endif

src/mlpack/methods/ann/layer/concat_performance.hpp

@@ -24,17 +24,17 @@ namespace ann /** Artificial Neural Network. */ {
  * feed-forward fully connected network container which plugs performance layers
  * together.
  *
- * @tparam InputType Type of the input data (arma::colvec, arma::mat,
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
- * @tparam OutputType Type of the output data (arma::colvec, arma::mat,
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
  */
 template <
     typename OutputLayerType = NegativeLogLikelihood<>,
-    typename InputType = arma::mat,
-    typename OutputType = arma::mat
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
 >
-class ConcatPerformance : public Layer<InputType, OutputType>
+class ConcatPerformance
 {
  public:
   /**
@@ -43,7 +43,8 @@ class ConcatPerformance : public Layer<InputType, OutputType>
    * @param inSize The number of inputs.
    * @param outputLayer Output layer used to evaluate the network.
    */
-  ConcatPerformance(OutputLayerType&& outputLayer = OutputLayerType());
+  ConcatPerformance(const size_t inSize = 0,
+                    OutputLayerType&& outputLayer = OutputLayerType());
 
   /*
    * Computes the Negative log likelihood.
@@ -51,7 +52,8 @@ class ConcatPerformance : public Layer<InputType, OutputType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
-  void Forward(const InputType& input, OutputType& target);
+  template<typename eT>
+  double Forward(const arma::Mat<eT>& input, arma::Mat<eT>& target);
 
   /**
    * Ordinary feed backward pass of a neural network. The negative log
@@ -64,29 +66,42 @@ class ConcatPerformance : public Layer<InputType, OutputType>
    *        between 1 and the number of classes.
    * @param output The calculated error.
    */
-  void Backward(const InputType& input,
-                const OutputType& target,
-                OutputType& output);
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& input,
+                const arma::Mat<eT>& target,
+                arma::Mat<eT>& output);
 
   //! Get the output parameter.
-  OutputType& OutputParameter() const { return outputParameter; }
+  OutputDataType& OutputParameter() const { return outputParameter; }
   //! Modify the output parameter.
-  OutputType& OutputParameter() { return outputParameter; }
+  OutputDataType& OutputParameter() { return outputParameter; }
 
   //! Get the delta.
-  OutputType& Delta() const { return delta; }
+  OutputDataType& Delta() const { return delta; }
   //! Modify the delta.
-  OutputType& Delta() { return delta; }
+  OutputDataType& Delta() { return delta; }
+
+  //! Get the number of inputs.
+  size_t InSize() const { return inSize; }
 
   /**
-   * Serialize the layer.
+   * Serialize the layer
    */
   template<typename Archive>
   void serialize(Archive& /* ar */, const uint32_t /* version */);
 
  private:
+  //! Locally-stored number of inputs.
+  size_t inSize;
+
   //! Instantiated outputlayer used to evaluate the network.
   OutputLayerType outputLayer;
+
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
 }; // class ConcatPerformance
 
 } // namespace ann

src/mlpack/methods/ann/layer/concat_performance_impl.hpp

@@ -20,14 +20,15 @@ namespace ann /** Artificial Neural Network. */ {
 
 template<
     typename OutputLayerType,
-    typename InputType,
-    typename OutputType
+    typename InputDataType,
+    typename OutputDataType
 >
 ConcatPerformance<
     OutputLayerType,
-    InputType,
-    OutputType
->::ConcatPerformance(OutputLayerType&& outputLayer) :
+    InputDataType,
+    OutputDataType
+>::ConcatPerformance(const size_t inSize, OutputLayerType&& outputLayer) :
+    inSize(inSize),
     outputLayer(std::move(outputLayer))
 {
   // Nothing to do here.
@@ -35,51 +36,51 @@ ConcatPerformance<
 
 template<
     typename OutputLayerType,
-    typename InputType,
-    typename OutputType
+    typename InputDataType,
+    typename OutputDataType
 >
-void ConcatPerformance<
+template<typename eT>
+double ConcatPerformance<
     OutputLayerType,
-    InputType,
-    OutputType
->::Forward(const InputType& input, OutputType& target)
+    InputDataType,
+    OutputDataType
+>::Forward(const arma::Mat<eT>& input, arma::Mat<eT>& target)
 {
-  const size_t elements = input.n_elem / inputDimensions[0];
+  const size_t elements = input.n_elem / inSize;
 
   double output = 0;
-  for (size_t i = 0; i < input.n_elem; i += elements)
+  for (size_t i = 0; i < input.n_elem; i+= elements)
   {
-    InputType subInput = input.submat(i, 0, i + elements - 1, 0);
+    arma::mat subInput = input.submat(i, 0, i + elements - 1, 0);
     output += outputLayer.Forward(subInput, target);
   }
 
-  // TODO: what to do with output?
-  //return output;
-  return;
+  return output;
 }
 
 template<
     typename OutputLayerType,
-    typename InputType,
-    typename OutputType
+    typename InputDataType,
+    typename OutputDataType
 >
+template<typename eT>
 void ConcatPerformance<
     OutputLayerType,
-    InputType,
-    OutputType
+    InputDataType,
+    OutputDataType
 >::Backward(
-    const InputType& input,
-    const OutputType& target,
-    OutputType& output)
+    const arma::Mat<eT>& input,
+    const arma::Mat<eT>& target,
+    arma::Mat<eT>& output)
 {
-  const size_t elements = input.n_elem / inputDimensions[0];
+  const size_t elements = input.n_elem / inSize;
 
-  InputType subInput = input.submat(0, 0, elements - 1, 0);
-  OutputType subOutput;
+  arma::mat subInput = input.submat(0, 0, elements - 1, 0);
+  arma::mat subOutput;
 
   outputLayer.Backward(subInput, target, subOutput);
 
-  output = arma::zeros(subOutput.n_elem, inputDimensions[0]);
+  output = arma::zeros(subOutput.n_elem, inSize);
   output.col(0) = subOutput;
 
   for (size_t i = elements, j = 0; i < input.n_elem; i+= elements, ++j)
@@ -93,19 +94,17 @@ void ConcatPerformance<
 
 template<
     typename OutputLayerType,
-    typename InputType,
-    typename OutputType
+    typename InputDataType,
+    typename OutputDataType
 >
 template<typename Archive>
 void ConcatPerformance<
     OutputLayerType,
-    InputType,
-    OutputType
+    InputDataType,
+    OutputDataType
 >::serialize(Archive& ar, const uint32_t /* version */)
 {
-  ar(cereal::base_class<Layer<InputType, OutputType>>(this));
-
-  ar(CEREAL_NVP(outputLayer));
+  ar(CEREAL_NVP(inSize));
 }
 
 } // namespace ann

src/mlpack/methods/ann/layer/concatenate.hpp

@@ -14,7 +14,7 @@
 #define MLPACK_METHODS_ANN_LAYER_CONCATENATE_HPP
 
 #include <mlpack/prereqs.hpp>
-#include "layer.hpp"
+#include <mlpack/methods/ann/layer/layer_traits.hpp>
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
@@ -22,39 +22,36 @@ namespace ann /** Artificial Neural Network. */ {
 /**
  * Implementation of the Concatenate module class. The Concatenate module
  * concatenates a constant given matrix to the incoming data.
+ * Note: Users need to use the Concat() function to provide the concat matrix.
  *
- * The Concat() function provides the concat matrix, or it can be passed to
- * the constructor.
- *
- * After this layer is applied, the shape of the data will be a vector.
- *
- * @tparam MatType Matrix representation to accept as input and use for
- *    computation.
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
  */
-template<typename MatType = arma::mat>
-class ConcatenateType : public Layer<MatType>
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+class Concatenate
 {
  public:
   /**
-   * Create the ConcatenateType object using the given constant matrix as the
-   * data to be concatenated to the output of the forward pass.
+   * Create the Concatenate object using the specified number of output units.
    */
-  ConcatenateType(const MatType& concat = MatType());
+  Concatenate();
 
-  //! Clone the ConcatenateType object. This handles polymorphism correctly.
-  ConcatenateType* Clone() const { return new ConcatenateType(*this); }
+  //! Copy constructor.
+  Concatenate(const Concatenate& layer);
 
-  // Virtual destructor.
-  virtual ~ConcatenateType() { }
+  //! Move constructor.
+  Concatenate(Concatenate&& layer);
 
-  //! Copy the given ConcatenateType layer.
-  ConcatenateType(const ConcatenateType& other);
-  //! Take ownership of the given ConcatenateType layer.
-  ConcatenateType(ConcatenateType&& other);
-  //! Copy the given ConcatenateType layer.
-  ConcatenateType& operator=(const ConcatenateType& other);
-  //! Take ownership of the given ConcatenateType layer.
-  ConcatenateType& operator=(ConcatenateType&& other);
+  //! Operator= copy constructor.
+  Concatenate& operator=(const Concatenate& layer);
+
+  //! Operator= move constructor.
+  Concatenate& operator=(Concatenate&& layer);
 
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function
@@ -63,7 +60,8 @@ class ConcatenateType : public Layer<MatType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
-  void Forward(const MatType& input, MatType& output);
+  template<typename eT>
+  void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);
 
   /**
    * Ordinary feed backward pass of a neural network, calculating the function
@@ -74,31 +72,57 @@ class ConcatenateType : public Layer<MatType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const MatType& /* input */, const MatType& gy, MatType& g);
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& /* input */,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g);
+
+  //! Get the parameters.
+  OutputDataType const& Parameters() const { return weights; }
+  //! Modify the parameters.
+  OutputDataType& Parameters() { return weights; }
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
 
   //! Get the concat matrix.
-  MatType const& Concat() const { return concat; }
+  OutputDataType const& Concat() const { return concat; }
   //! Modify the concat.
-  MatType& Concat() { return concat; }
-
-  //! Compute the output dimensions of the layer based on `InputDimensions()`.
-  void ComputeOutputDimensions();
+  OutputDataType& Concat() { return concat; }
 
   /**
-   * Serialize the layer.
+   * Serialize the layer
    */
   template<typename Archive>
-  void serialize(Archive& ar, const uint32_t /* version */);
+  void serialize(Archive& /* ar */, const uint32_t /* version */)
+  {
+    // Nothing to do here.
+  }
 
  private:
-  //! Matrix to be concatenated to input.
-  MatType concat;
+  //! Locally-stored number of input rows.
+  size_t inRows;
 
+  //! Locally-stored weight object.
+  OutputDataType weights;
+
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
+  //! Locally-stored matrix to be concatenated to input.
+  OutputDataType concat;
 }; // class Concatenate
 
-// Standard Concatenate layer.
-typedef ConcatenateType<arma::mat> Concatenate;
-
 } // namespace ann
 } // namespace mlpack
 

src/mlpack/methods/ann/layer/concatenate_impl.hpp

@@ -19,106 +19,91 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template<typename MatType>
-ConcatenateType<MatType>::
-ConcatenateType(const MatType& concat) :
-    concat(concat)
+template<typename InputDataType, typename OutputDataType>
+Concatenate<InputDataType, OutputDataType>::Concatenate() :
+    inRows(0)
 {
   // Nothing to do here.
 }
 
-template<typename MatType>
-ConcatenateType<MatType>::
-ConcatenateType(const ConcatenateType& other) :
-    Layer<MatType>(other),
-    concat(other.concat)
+template<typename InputDataType, typename OutputDataType>
+Concatenate<InputDataType, OutputDataType>::Concatenate(
+    const Concatenate& layer) :
+    inRows(layer.inRows),
+    weights(layer.weights),
+    delta(layer.delta),
+    concat(layer.concat)
 {
-  // Nothing to do.
+  // Nothing to to here.
 }
 
-template<typename MatType>
-ConcatenateType<MatType>::
-ConcatenateType(ConcatenateType&& other) :
-    Layer<MatType>(std::move(other)),
-    concat(other.concat)
+template<typename InputDataType, typename OutputDataType>
+Concatenate<InputDataType, OutputDataType>::Concatenate(Concatenate&& layer) :
+    inRows(layer.inRows),
+    weights(std::move(layer.weights)),
+    delta(std::move(layer.delta)),
+    concat(std::move(layer.concat))
 {
-  // Nothing to do.
+  // Nothing to do here.
 }
 
-template<typename MatType>
-ConcatenateType<MatType>&
-ConcatenateType<MatType>::operator=(const ConcatenateType& other)
+template<typename InputDataType, typename OutputDataType>
+Concatenate<InputDataType, OutputDataType>&
+Concatenate<InputDataType, OutputDataType>::
+operator=(const Concatenate& layer)
 {
-  if (&other != this)
+  if (this != &layer)
   {
-    Layer<MatType>::operator=(other);
-    concat = other.concat;
+    inRows = layer.inRows;
+    weights = layer.weights;
+    delta = layer.delta;
+    concat = layer.concat;
   }
 
   return *this;
 }
 
-template<typename MatType>
-ConcatenateType<MatType>&
-ConcatenateType<MatType>::operator=(ConcatenateType&& other)
+template<typename InputDataType, typename OutputDataType>
+Concatenate<InputDataType, OutputDataType>&
+Concatenate<InputDataType, OutputDataType>::
+operator=(Concatenate&& layer)
 {
-  if (&other != this)
+  if (this != &layer)
   {
-    Layer<MatType>::operator=(std::move(other));
-    concat = std::move(other.concat);
+    inRows = layer.inRows;
+    weights = std::move(layer.weights);
+    delta = std::move(layer.delta);
+    concat = std::move(layer.concat);
   }
-
   return *this;
 }
 
-template<typename MatType>
-void ConcatenateType<MatType>::Forward(const MatType& input, MatType& output)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void Concatenate<InputDataType, OutputDataType>::Forward(
+    const arma::Mat<eT>& input, arma::Mat<eT>& output)
 {
   if (concat.is_empty())
+    Log::Warn << "The concat matrix has not been provided." << std::endl;
+
+  if (input.n_cols != concat.n_cols)
   {
-    Log::Warn << "Concatenate::Forward(): the concat matrix is empty or was "
-        << "not provided." << std::endl;
+    Log::Fatal << "The number of columns of the concat matrix should be equal "
+        << "to the number of columns of input matrix." << std::endl;
   }
 
-  output.submat(0, 0, input.n_rows - 1, input.n_cols - 1) = input;
-  output.submat(input.n_rows, 0, output.n_rows - 1, input.n_cols - 1) =
-      arma::repmat(arma::vectorise(concat), 1, input.n_cols);
+  inRows = input.n_rows;
+  output = arma::join_cols(input, concat);
 }
 
-template<typename MatType>
-void ConcatenateType<MatType>::Backward(
-    const MatType& /* input */,
-    const MatType& gy,
-    MatType& g)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void Concatenate<InputDataType, OutputDataType>::Backward(
+    const arma::Mat<eT>& /* input */,
+    const arma::Mat<eT>& gy,
+    arma::Mat<eT>& g)
 {
-  // Pass back the non-concatenated part.
-  g = gy.submat(0, 0, gy.n_rows - 1 - concat.n_elem, gy.n_cols - 1);
-}
-
-template<typename MatType>
-void ConcatenateType<MatType>::ComputeOutputDimensions()
-{
-  // This flattens the input.
-  size_t inSize = this->inputDimensions[0];
-  for (size_t i = 1; i < this->inputDimensions.size(); ++i)
-      inSize *= this->inputDimensions[i];
-
-  this->outputDimensions = std::vector<size_t>(this->inputDimensions.size(),
-      1);
-  this->outputDimensions[0] = inSize + concat.n_elem;
-}
-
-/**
- * Serialize the layer.
- */
-template<typename MatType>
-template<typename Archive>
-void ConcatenateType<MatType>::serialize(
-  Archive& ar, const uint32_t /* version */)
-{
-  ar(cereal::base_class<Layer<MatType>>(this));
-
-  ar(CEREAL_NVP(concat));
+  g = gy.submat(0, 0, inRows - 1, concat.n_cols - 1);
 }
 
 } // namespace ann

src/mlpack/methods/ann/layer/constant.hpp

@@ -15,8 +15,6 @@
 
 #include <mlpack/prereqs.hpp>
 
-#include "layer.hpp"
-
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
@@ -24,21 +22,18 @@ namespace ann /** Artificial Neural Network. */ {
  * Implementation of the constant layer. The constant layer outputs a given
  * constant value given any input value.
  *
- * @tparam InputType The type of the layer's inputs. The layer automatically
- *     cast inputs to this type (Default: arma::mat).
- * @tparam OutputType The type of the computation which also causes the output
- *     to also be in this type. The type also allows the computation and weight
- *     type to differ from the input type (Default: arma::mat).
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
  */
-template<typename InputType = arma::mat, typename OutputType = arma::mat>
-class ConstantType : public Layer<InputType, OutputType>
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+class Constant
 {
  public:
-  /**
-   * Create an empty Constant layer.
-   */
-  ConstantType();
-
   /**
    * Create the Constant object that outputs a given constant scalar value
    * given any input value.
@@ -46,19 +41,7 @@ class ConstantType : public Layer<InputType, OutputType>
    * @param outSize The number of output units.
    * @param scalar The constant value used to create the constant output.
    */
-  ConstantType(const size_t outSize, const double scalar = 0);
-
-  //! Copy another ConstantType.
-  ConstantType(const ConstantType& layer);
-  //! Take ownership of another ConstantType.
-  ConstantType(ConstantType&& layer);
-  //! Copy another ConstantType.
-  ConstantType& operator=(const ConstantType& layer);
-  //! Take ownership of another ConstantType.
-  ConstantType& operator=(ConstantType&& layer);
-
-  //! Clone the ConstantType object. This handles polymorphism correctly.
-  ConstantType* Clone() const { return new ConstantType(*this); }
+  Constant(const size_t outSize = 0, const double scalar = 0.0);
 
   /**
    * Ordinary feed forward pass of a neural network. The forward pass fills the
@@ -67,6 +50,7 @@ class ConstantType : public Layer<InputType, OutputType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
+  template<typename InputType, typename OutputType>
   void Forward(const InputType& input, OutputType& output);
 
   /**
@@ -77,34 +61,52 @@ class ConstantType : public Layer<InputType, OutputType>
    * @param * (gy) The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const InputType& /* input */,
-                const OutputType& /* gy */,
-                OutputType& g);
+  template<typename DataType>
+  void Backward(const DataType& /* input */,
+                const DataType& /* gy */,
+                DataType& g);
+
+  //! Get the output parameter.
+  OutputDataType& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
 
   //! Get the output size.
-  const std::vector<size_t>& OutputDimensions() const
+  size_t OutSize() const { return outSize; }
+
+  //! Get the size of the weights.
+  size_t WeightSize() const
   {
-    std::vector<size_t> result(this->inputDimensions.size(), 0);
-    result[0] = outSize;
-    return result;
+    return 0;
   }
 
-  //! Serialize the layer.
+  /**
+   * Serialize the layer.
+   */
   template<typename Archive>
   void serialize(Archive& ar, const uint32_t /* version */);
 
  private:
+  //! Locally-stored number of input units.
+  size_t inSize;
+
   //! Locally-stored number of output units.
   size_t outSize;
 
   //! Locally-stored constant output matrix.
-  OutputType constantOutput;
-}; // class ConstantType
+  OutputDataType constantOutput;
 
-// Convenience typedefs.
+  //! Locally-stored delta object.
+  OutputDataType delta;
 
-// Standard HardShrink layer.
-typedef ConstantType<arma::mat, arma::mat> Constant;
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+}; // class ConstantLayer
 
 } // namespace ann
 } // namespace mlpack

src/mlpack/methods/ann/layer/constant_impl.hpp

@@ -0,0 +1,65 @@
+/**
+ * @file methods/ann/layer/constant_impl.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of the Constant class, which outputs a constant value given
+ * any input.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_CONSTANT_IMPL_HPP
+#define MLPACK_METHODS_ANN_LAYER_CONSTANT_IMPL_HPP
+
+// In case it hasn't yet been included.
+#include "constant.hpp"
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+template<typename InputDataType, typename OutputDataType>
+Constant<InputDataType, OutputDataType>::Constant(
+    const size_t outSize,
+    const double scalar) :
+    inSize(0),
+    outSize(outSize)
+{
+  constantOutput = OutputDataType(outSize, 1);
+  constantOutput.fill(scalar);
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename InputType, typename OutputType>
+void Constant<InputDataType, OutputDataType>::Forward(
+    const InputType& input, OutputType& output)
+{
+  if (inSize == 0)
+  {
+    inSize = input.n_elem;
+  }
+
+  output = constantOutput;
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename DataType>
+void Constant<InputDataType, OutputDataType>::Backward(
+    const DataType& /* input */, const DataType& /* gy */, DataType& g)
+{
+  g = arma::zeros<DataType>(inSize, 1);
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename Archive>
+void Constant<InputDataType, OutputDataType>::serialize(
+    Archive& ar, const uint32_t /* version */)
+{
+  ar(CEREAL_NVP(constantOutput));
+}
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/layer/convolution.hpp

@@ -20,7 +20,7 @@
 #include <mlpack/methods/ann/convolution_rules/svd_convolution.hpp>
 #include <mlpack/core/util/to_lower.hpp>
 
-#include "layer.hpp"
+#include "layer_types.hpp"
 #include "padding.hpp"
 
 namespace mlpack {
@@ -30,7 +30,7 @@ namespace ann /** Artificial Neural Network. */ {
  * Implementation of the Convolution class. The Convolution class represents a
  * single layer of a neural network.
  * Example usage:
- *
+ * 
  * Suppose we want to pass a matrix M (2744x100) to a `Convolution` layer;
  * in this example, `M` was obtained from "flattening" 100 images (or Mel
  * cepstral coefficients, if we talk about speech, or whatever you like) of
@@ -62,95 +62,100 @@ namespace ann /** Artificial Neural Network. */ {
  * @tparam ForwardConvolutionRule Convolution to perform forward process.
  * @tparam BackwardConvolutionRule Convolution to perform backward process.
  * @tparam GradientConvolutionRule Convolution to calculate gradient.
- * @tparam MatType Matrix representation to accept as input and use for
- *    computation.
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
  */
 template <
     typename ForwardConvolutionRule = NaiveConvolution<ValidConvolution>,
     typename BackwardConvolutionRule = NaiveConvolution<FullConvolution>,
     typename GradientConvolutionRule = NaiveConvolution<ValidConvolution>,
-    typename MatType = arma::mat
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
 >
-class ConvolutionType : public Layer<MatType>
+class Convolution
 {
  public:
-  //! Create the ConvolutionType object.
-  ConvolutionType();
+  //! Create the Convolution object.
+  Convolution();
 
   /**
-   * Create the ConvolutionType object using the specified number of output
-   * maps, filter size, stride and padding parameter.
+   * Create the Convolution object using the specified number of input maps,
+   * output maps, filter size, stride and padding parameter.
    *
-   * @param maps The number of output maps.
+   * @param inSize The number of input maps.
+   * @param outSize The number of output maps.
    * @param kernelWidth Width of the filter/kernel.
    * @param kernelHeight Height of the filter/kernel.
    * @param strideWidth Stride of filter application in the x direction.
    * @param strideHeight Stride of filter application in the y direction.
    * @param padW Padding width of the input.
    * @param padH Padding height of the input.
-   * @param paddingType The type of padding ("valid" or "same"). Defaults to
-   *    "none".  If not specified or "none", the values for `padW` and `padH`
-   *    will be used.
+   * @param inputWidth The width of the input data.
+   * @param inputHeight The height of the input data.
+   * @param paddingType The type of padding (Valid or Same). Defaults to None.
    */
-  ConvolutionType(const size_t maps,
-                  const size_t kernelWidth,
-                  const size_t kernelHeight,
-                  const size_t strideWidth = 1,
-                  const size_t strideHeight = 1,
-                  const size_t padW = 0,
-                  const size_t padH = 0,
-                  const std::string& paddingType = "none");
+  Convolution(const size_t inSize,
+              const size_t outSize,
+              const size_t kernelWidth,
+              const size_t kernelHeight,
+              const size_t strideWidth = 1,
+              const size_t strideHeight = 1,
+              const size_t padW = 0,
+              const size_t padH = 0,
+              const size_t inputWidth = 0,
+              const size_t inputHeight = 0,
+              const std::string& paddingType = "None");
 
   /**
    * Create the Convolution object using the specified number of input maps,
    * output maps, filter size, stride and padding parameter.
    *
-   * @param maps The number of output maps.
+   * @param inSize The number of input maps.
+   * @param outSize The number of output maps.
    * @param kernelWidth Width of the filter/kernel.
    * @param kernelHeight Height of the filter/kernel.
    * @param strideWidth Stride of filter application in the x direction.
    * @param strideHeight Stride of filter application in the y direction.
-   * @param padW A two-value tuple indicating padding widths of the input.  The
-   *      first value is the padding for the left side; the second value is the
-   *      padding on the right side.
-   * @param padH A two-value tuple indicating padding heights of the input.  The
-   *      first value is the padding for the top; the second value is the
-   *      padding on the bottom.
-   * @param paddingType The type of padding ("valid" or "same"). Defaults to
-   *      "none".  If not specified or "none", the values for `padW` and `padH`
-   *      will be used.
+   * @param padW A two-value tuple indicating padding widths of the input.
+   *             First value is padding at left side. Second value is padding on
+   *             right side.
+   * @param padH A two-value tuple indicating padding heights of the input.
+   *             First value is padding at top. Second value is padding on
+   *             bottom.
+   * @param inputWidth The width of the input data.
+   * @param inputHeight The height of the input data.
+   * @param paddingType The type of padding (Valid or Same). Defaults to None.
    */
-  ConvolutionType(const size_t maps,
-                  const size_t kernelWidth,
-                  const size_t kernelHeight,
-                  const size_t strideWidth,
-                  const size_t strideHeight,
-                  const std::tuple<size_t, size_t>& padW,
-                  const std::tuple<size_t, size_t>& padH,
-                  const std::string& paddingType = "none");
+  Convolution(const size_t inSize,
+              const size_t outSize,
+              const size_t kernelWidth,
+              const size_t kernelHeight,
+              const size_t strideWidth,
+              const size_t strideHeight,
+              const std::tuple<size_t, size_t>& padW,
+              const std::tuple<size_t, size_t>& padH,
+              const size_t inputWidth = 0,
+              const size_t inputHeight = 0,
+              const std::string& paddingType = "None");
 
-  //! Clone the ConvolutionType object. This handles polymorphism correctly.
-  ConvolutionType* Clone() const { return new ConvolutionType(*this); }
+  //! Copy constructor.
+  Convolution(const Convolution& layer);
 
-  //! Copy the given ConvolutionType (but not weights).
-  ConvolutionType(const ConvolutionType& layer);
+  //! Move constructor.
+  Convolution(Convolution&&);
 
-  //! Take ownership of the given ConvolutionType (but not weights).
-  ConvolutionType(ConvolutionType&&);
+  //! Copy assignment operator.
+  Convolution& operator=(const Convolution& layer);
 
-  //! Copy the given ConvolutionType (but not weights).
-  ConvolutionType& operator=(const ConvolutionType& layer);
-
-  //! Take ownership of the given ConvolutionType (but not weights).
-  ConvolutionType& operator=(ConvolutionType&& layer);
-
-  // Virtual destructor.
-  virtual ~ConvolutionType() { }
+  //! Move assignment operator.
+  Convolution& operator=(Convolution&& layer);
 
   /*
    * Set the weight and bias term.
    */
-  void SetWeights(typename MatType::elem_type* weightsPtr);
+  void Reset();
 
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function
@@ -159,7 +164,8 @@ class ConvolutionType : public Layer<MatType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
-  void Forward(const MatType& input, MatType& output);
+  template<typename eT>
+  void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);
 
   /**
    * Ordinary feed backward pass of a neural network, calculating the function
@@ -170,91 +176,135 @@ class ConvolutionType : public Layer<MatType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const MatType& /* input */,
-                const MatType& gy,
-                MatType& g);
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& /* input */,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g);
 
-  /**
+  /*
    * Calculate the gradient using the output delta and the input activation.
    *
    * @param input The input parameter used for calculating the gradient.
    * @param error The calculated error.
    * @param gradient The calculated gradient.
    */
-  void Gradient(const MatType& /* input */,
-                const MatType& error,
-                MatType& gradient);
+  template<typename eT>
+  void Gradient(const arma::Mat<eT>& /* input */,
+                const arma::Mat<eT>& error,
+                arma::Mat<eT>& gradient);
 
   //! Get the parameters.
-  MatType const& Parameters() const { return weights; }
+  OutputDataType const& Parameters() const { return weights; }
   //! Modify the parameters.
-  MatType& Parameters() { return weights; }
+  OutputDataType& Parameters() { return weights; }
 
-  //! Get the weight of the layer as a cube.
-  arma::Cube<typename MatType::elem_type> const& Weight() const
-  {
-    return weight;
-  }
-  //! Modify the weight of the layer as a cube.
-  arma::Cube<typename MatType::elem_type>& Weight() { return weight; }
+  //! Get the weight of the layer.
+  arma::cube const& Weight() const { return weight; }
+  //! Modify the weight of the layer.
+  arma::cube& Weight() { return weight; }
 
   //! Get the bias of the layer.
-  MatType const& Bias() const { return bias; }
+  arma::mat const& Bias() const { return bias; }
   //! Modify the bias of the layer.
-  MatType& Bias() { return bias; }
+  arma::mat& Bias() { return bias; }
+
+  //! Get the input parameter.
+  InputDataType const& InputParameter() const { return inputParameter; }
+  //! Modify the input parameter.
+  InputDataType& InputParameter() { return inputParameter; }
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  //! Get the gradient.
+  OutputDataType const& Gradient() const { return gradient; }
+  //! Modify the gradient.
+  OutputDataType& Gradient() { return gradient; }
+
+  //! Get the input width.
+  size_t InputWidth() const { return inputWidth; }
+  //! Modify input the width.
+  size_t& InputWidth() { return inputWidth; }
+
+  //! Get the input height.
+  size_t InputHeight() const { return inputHeight; }
+  //! Modify the input height.
+  size_t& InputHeight() { return inputHeight; }
+
+  //! Get the output width.
+  size_t OutputWidth() const { return outputWidth; }
+  //! Modify the output width.
+  size_t& OutputWidth() { return outputWidth; }
+
+  //! Get the output height.
+  size_t OutputHeight() const { return outputHeight; }
+  //! Modify the output height.
+  size_t& OutputHeight() { return outputHeight; }
+
+  //! Get the number of input maps.
+  size_t InputSize() const { return inSize; }
 
   //! Get the number of output maps.
-  size_t const& Maps() const { return maps; }
+  size_t OutputSize() const { return outSize; }
 
   //! Get the kernel width.
-  size_t const& KernelWidth() const { return kernelWidth; }
+  size_t KernelWidth() const { return kernelWidth; }
   //! Modify the kernel width.
   size_t& KernelWidth() { return kernelWidth; }
 
   //! Get the kernel height.
-  size_t const& KernelHeight() const { return kernelHeight; }
+  size_t KernelHeight() const { return kernelHeight; }
   //! Modify the kernel height.
   size_t& KernelHeight() { return kernelHeight; }
 
   //! Get the stride width.
-  size_t const& StrideWidth() const { return strideWidth; }
+  size_t StrideWidth() const { return strideWidth; }
   //! Modify the stride width.
   size_t& StrideWidth() { return strideWidth; }
 
   //! Get the stride height.
-  size_t const& StrideHeight() const { return strideHeight; }
+  size_t StrideHeight() const { return strideHeight; }
   //! Modify the stride height.
   size_t& StrideHeight() { return strideHeight; }
 
   //! Get the top padding height.
-  size_t const& PadHTop() const { return padHTop; }
+  size_t PadHTop() const { return padHTop; }
   //! Modify the top padding height.
   size_t& PadHTop() { return padHTop; }
 
   //! Get the bottom padding height.
-  size_t const& PadHBottom() const { return padHBottom; }
+  size_t PadHBottom() const { return padHBottom; }
   //! Modify the bottom padding height.
   size_t& PadHBottom() { return padHBottom; }
 
   //! Get the left padding width.
-  size_t const& PadWLeft() const { return padWLeft; }
+  size_t PadWLeft() const { return padWLeft; }
   //! Modify the left padding width.
   size_t& PadWLeft() { return padWLeft; }
 
   //! Get the right padding width.
-  size_t const& PadWRight() const { return padWRight; }
+  size_t PadWRight() const { return padWRight; }
   //! Modify the right padding width.
   size_t& PadWRight() { return padWRight; }
 
   //! Get size of weights for the layer.
   size_t WeightSize() const
   {
-    return (maps * inMaps * higherInDimensions * kernelWidth * kernelHeight) +
-        maps;
+    return (outSize * inSize * kernelWidth * kernelHeight) + outSize;
   }
 
-  //! Compute the output dimensions of the layer based on `InputDimensions()`.
-  void ComputeOutputDimensions();
+  //! Get the shape of the input.
+  size_t InputShape() const
+  {
+    return inputHeight * inputWidth * inSize;
+  }
 
   /**
    * Serialize the layer.
@@ -263,7 +313,7 @@ class ConvolutionType : public Layer<MatType>
   void serialize(Archive& ar, const uint32_t /* version */);
 
  private:
-  /**
+  /*
    * Return the convolution output size.
    *
    * @param size The size of the input (row or column).
@@ -282,12 +332,12 @@ class ConvolutionType : public Layer<MatType>
     return std::floor(size + pSideOne + pSideTwo - k) / s + 1;
   }
 
-  /**
+  /*
    * Function to assign padding such that output size is same as input size.
    */
   void InitializeSamePadding();
 
-  /**
+  /*
    * Rotates a 3rd-order tensor counterclockwise by 180 degrees.
    *
    * @param input The input data to be rotated.
@@ -303,7 +353,7 @@ class ConvolutionType : public Layer<MatType>
       output.slice(s) = arma::fliplr(arma::flipud(input.slice(s)));
   }
 
-  /**
+  /*
    * Rotates a dense matrix counterclockwise by 180 degrees.
    *
    * @param input The input data to be rotated.
@@ -316,8 +366,11 @@ class ConvolutionType : public Layer<MatType>
     output = arma::fliplr(arma::flipud(input));
   }
 
+  //! Locally-stored number of input channels.
+  size_t inSize;
+
   //! Locally-stored number of output channels.
-  size_t maps;
+  size_t outSize;
 
   //! Locally-stored number of input units.
   size_t batchSize;
@@ -347,46 +400,54 @@ class ConvolutionType : public Layer<MatType>
   size_t padHTop;
 
   //! Locally-stored weight object.
-  MatType weights;
+  OutputDataType weights;
 
   //! Locally-stored weight object.
-  arma::Cube<typename MatType::elem_type> weight;
+  arma::cube weight;
 
   //! Locally-stored bias term object.
-  MatType bias;
+  arma::mat bias;
+
+  //! Locally-stored input width.
+  size_t inputWidth;
+
+  //! Locally-stored input height.
+  size_t inputHeight;
+
+  //! Locally-stored output width.
+  size_t outputWidth;
+
+  //! Locally-stored output height.
+  size_t outputHeight;
 
   //! Locally-stored transformed output parameter.
-  arma::Cube<typename MatType::elem_type> outputTemp;
+  arma::cube outputTemp;
 
   //! Locally-stored transformed padded input parameter.
-  MatType inputPadded;
+  arma::cube inputPaddedTemp;
 
   //! Locally-stored transformed error parameter.
-  arma::Cube<typename MatType::elem_type> gTemp;
+  arma::cube gTemp;
 
   //! Locally-stored transformed gradient parameter.
-  arma::Cube<typename MatType::elem_type> gradientTemp;
+  arma::cube gradientTemp;
 
   //! Locally-stored padding layer.
-  ann::Padding padding;
+  ann::Padding<> padding;
 
-  //! Type of padding.
-  std::string paddingType;
+  //! Locally-stored delta object.
+  OutputDataType delta;
 
-  //! Locally-cached number of input maps.
-  size_t inMaps;
-  //! Locally-cached higher-order input dimensions.
-  size_t higherInDimensions;
+  //! Locally-stored gradient object.
+  OutputDataType gradient;
+
+  //! Locally-stored input parameter object.
+  InputDataType inputParameter;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
 }; // class Convolution
 
-// Standard Convolution layer.
-typedef ConvolutionType<
-    NaiveConvolution<ValidConvolution>,
-    NaiveConvolution<FullConvolution>,
-    NaiveConvolution<ValidConvolution>,
-    arma::mat
-> Convolution;
-
 } // namespace ann
 } // namespace mlpack
 

src/mlpack/methods/ann/layer/dropconnect.hpp

@@ -16,7 +16,10 @@
 
 #include <mlpack/prereqs.hpp>
 
-#include "layer.hpp"
+#include "layer_types.hpp"
+#include "add_merge.hpp"
+#include "linear.hpp"
+#include "sequential.hpp"
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
@@ -25,58 +28,55 @@ namespace ann /** Artificial Neural Network. */ {
  * The DropConnect layer is a regularizer that randomly with probability
  * ratio sets the connection values to zero and scales the remaining
  * elements by factor 1 /(1 - ratio). The output is scaled with 1 / (1 - p)
- * when in training mode.  During testing, the layer just computes the output.
- * The output is computed according to the input layer. If no input layer is
- * given, it will take a linear layer as default.
+ * when deterministic is false. In the deterministic mode(during testing),
+ * the layer just computes the output. The output is computed according
+ * to the input layer. If no input layer is given, it will take a linear layer
+ * as default.
  *
- * For more information, see the following.
+ * Note:
+ * During training you should set deterministic to false and during testing
+ * you should set deterministic to true.
+ *
+ *  For more information, see the following.
  *
  * @code
  * @inproceedings{WanICML2013,
- *   title     = {Regularization of Neural Networks using DropConnect},
+ *   title={Regularization of Neural Networks using DropConnect},
  *   booktitle = {Proceedings of the 30th International Conference on Machine
  *                Learning(ICML - 13)},
- *   author    = {Li Wan and Matthew Zeiler and Sixin Zhang and Yann L. Cun and
- *                Rob Fergus},
- *   year      = {2013},
- *   url       = {http://proceedings.mlr.press/v28/wan13.pdf}
+ *   author = {Li Wan and Matthew Zeiler and Sixin Zhang and Yann L. Cun and
+ *             Rob Fergus},
+ *   year = {2013},
+ *   url  = {http://proceedings.mlr.press/v28/wan13.pdf}
  * }
  * @endcode
  *
- * @tparam MatType Matrix representation to accept as input and use for
- *    computation.
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
  */
-template<typename MatType = arma::mat>
-class DropConnectType : public Layer<MatType>
+template<
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+class DropConnect
 {
  public:
   //! Create the DropConnect object.
-  DropConnectType();
+  DropConnect();
 
   /**
-   * Creates the DropConnect Layer as a Linear Object that takes the number of
-   * output units and a ratio as parameter.
+   * Creates the DropConnect Layer as a Linear Object that takes input size,
+   * output size and ratio as parameter.
    *
+   * @param inSize The number of input units.
    * @param outSize The number of output units.
    * @param ratio The probability of setting a value to zero.
    */
-  DropConnectType(const size_t outSize,
-                  const double ratio = 0.5);
-
-  //! Clone the DropConnectType object. This handles polymorphism correctly.
-  DropConnectType* Clone() const { return new DropConnectType(*this); }
-
-  // Virtual destructor.
-  virtual ~DropConnectType();
-
-  //! Copy the given DropConnectType (except for weights).
-  DropConnectType(const DropConnectType& other);
-  //! Take ownership of the given DropConnectType (except for weights).
-  DropConnectType(DropConnectType&& other);
-  //! Copy the given DropConnectType (except for weights).
-  DropConnectType& operator=(const DropConnectType& other);
-  //! Take ownership of the given DropConnectType (except for weights).
-  DropConnectType& operator=(DropConnectType&& other);
+  DropConnect(const size_t inSize,
+              const size_t outSize,
+              const double ratio = 0.5);
 
   /**
    * Ordinary feed forward pass of the DropConnect layer.
@@ -84,7 +84,8 @@ class DropConnectType : public Layer<MatType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
-  void Forward(const MatType& input, MatType& output);
+  template<typename eT>
+  void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);
 
   /**
    * Ordinary feed backward pass of the DropConnect layer.
@@ -93,7 +94,10 @@ class DropConnectType : public Layer<MatType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const MatType& input, const MatType& gy, MatType& g);
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& input,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g);
 
   /**
    * Calculate the gradient using the output delta and the input activation.
@@ -102,7 +106,39 @@ class DropConnectType : public Layer<MatType>
    * @param error The calculated error.
    * @param * (gradient) The calculated gradient.
    */
-  void Gradient(const MatType& input, const MatType& error, MatType& gradient);
+  template<typename eT>
+  void Gradient(const arma::Mat<eT>& input,
+                const arma::Mat<eT>& error,
+                arma::Mat<eT>& /* gradient */);
+
+  //! Get the model modules.
+  std::vector<LayerTypes<> >& Model() { return network; }
+
+  //! Get the parameters.
+  OutputDataType const& Parameters() const { return weights; }
+  //! Modify the parameters.
+  OutputDataType& Parameters() { return weights; }
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  //! Get the gradient.
+  OutputDataType const& Gradient() const { return gradient; }
+  //! Modify the gradient.
+  OutputDataType& Gradient() { return gradient; }
+
+  //! The value of the deterministic parameter.
+  bool Deterministic() const { return deterministic; }
+
+  //! Modify the value of the deterministic parameter.
+  bool &Deterministic() { return deterministic; }
 
   //! The probability of setting a value to zero.
   double Ratio() const { return ratio; }
@@ -114,14 +150,8 @@ class DropConnectType : public Layer<MatType>
     scale = 1.0 / (1.0 - ratio);
   }
 
-  //! Compute the output dimensions of the layer based on `InputDimensions()`.
-  void ComputeOutputDimensions();
-
-  //! Return the size of the weights.
-  size_t WeightSize() const { return baseLayer->WeightSize(); }
-
-  // Set the weights to use the given memory `weightsPtr`.
-  void SetWeights(typename MatType::elem_type* weightsPtr);
+  //! Return the size of the weight matrix.
+  size_t WeightSize() const { return 0; }
 
   /**
    * Serialize the layer.
@@ -136,21 +166,34 @@ class DropConnectType : public Layer<MatType>
   //! The scale fraction.
   double scale;
 
+  //! Locally-stored weight object.
+  OutputDataType weights;
+
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored gradient object.
+  OutputDataType gradient;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
   //! Locally-stored mask object.
-  MatType mask;
+  OutputDataType mask;
+
+  //! If true dropout and scaling is disabled, see notes above.
+  bool deterministic;
 
   //! Denoise mask for the weights.
-  MatType denoise;
+  OutputDataType denoise;
 
   //! Locally-stored layer module.
-  Layer<MatType>* baseLayer;
+  LayerTypes<> baseLayer;
+
+  //! Locally-stored network modules.
+  std::vector<LayerTypes<> > network;
 }; // class DropConnect.
 
-// Convenience typedefs.
-
-// Standard DropConnect layer.
-typedef DropConnectType<arma::mat> DropConnect;
-
 }  // namespace ann
 }  // namespace mlpack
 

src/mlpack/methods/ann/layer/dropconnect_impl.hpp

@@ -17,160 +17,112 @@
 // In case it hasn't yet been included.
 #include "dropconnect.hpp"
 
-#include "linear.hpp"
+#include "../visitor/delete_visitor.hpp"
+#include "../visitor/forward_visitor.hpp"
+#include "../visitor/backward_visitor.hpp"
+#include "../visitor/gradient_visitor.hpp"
+#include "../visitor/parameters_set_visitor.hpp"
+#include "../visitor/parameters_visitor.hpp"
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template<typename MatType>
-DropConnectType<MatType>::DropConnectType() :
-    Layer<MatType>(),
+template<typename InputDataType, typename OutputDataType>
+DropConnect<InputDataType, OutputDataType>::DropConnect() :
     ratio(0.5),
     scale(2.0),
-    baseLayer(new LinearType<MatType>(0))
+    deterministic(true)
 {
   // Nothing to do here.
 }
 
-template<typename MatType>
-DropConnectType<MatType>::DropConnectType(
+template<typename InputDataType, typename OutputDataType>
+DropConnect<InputDataType, OutputDataType>::DropConnect(
+    const size_t inSize,
     const size_t outSize,
     const double ratio) :
-    Layer<MatType>(),
     ratio(ratio),
     scale(1.0 / (1 - ratio)),
-    baseLayer(new LinearType<MatType>(outSize))
+    baseLayer(new Linear<InputDataType, OutputDataType>(inSize, outSize))
 {
-  // Nothing to do.
+  network.push_back(baseLayer);
 }
 
-template<typename MatType>
-DropConnectType<MatType>::~DropConnectType()
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void DropConnect<InputDataType, OutputDataType>::Forward(
+    const arma::Mat<eT>& input,
+    arma::Mat<eT>& output)
 {
-  delete baseLayer;
-}
-
-template<typename MatType>
-DropConnectType<MatType>::DropConnectType(const DropConnectType& other) :
-    Layer<MatType>(other),
-    ratio(other.ratio),
-    scale(other.scale),
-    baseLayer(other.baseLayer->Clone())
-{
-  // Nothing to do.
-}
-
-template<typename MatType>
-DropConnectType<MatType>::DropConnectType(DropConnectType&& other) :
-    Layer<MatType>(std::move(other)),
-    ratio(std::move(other.ratio)),
-    scale(std::move(other.scale)),
-    baseLayer(std::move(other.baseLayer))
-{
-  // Nothing to do.
-}
-
-template<typename MatType>
-DropConnectType<MatType>&
-DropConnectType<MatType>::operator=(const DropConnectType& other)
-{
-  if (&other != this)
+  // The DropConnect mask will not be multiplied in the deterministic mode
+  // (during testing).
+  if (deterministic)
   {
-    Layer<MatType>::operator=(other);
-    ratio = other.ratio;
-    scale = other.scale;
-    baseLayer = other.baseLayer->Clone();
-  }
-
-  return *this;
-}
-
-template<typename MatType>
-DropConnectType<MatType>&
-DropConnectType<MatType>::operator=(DropConnectType&& other)
-{
-  if (&other != this)
-  {
-    Layer<MatType>::operator=(std::move(other));
-    ratio = std::move(other.ratio);
-    scale = std::move(other.scale);
-    baseLayer = std::move(other.baseLayer);
-  }
-
-  return *this;
-}
-
-template<typename MatType>
-void DropConnectType<MatType>::Forward(const MatType& input, MatType& output)
-{
-  // The DropConnect mask will not be multiplied in testing mode.
-  if (!this->training)
-  {
-    baseLayer->Forward(input, output);
+    boost::apply_visitor(ForwardVisitor(input, output), baseLayer);
   }
   else
   {
     // Save weights for denoising.
-    denoise = baseLayer->Parameters();
+    boost::apply_visitor(ParametersVisitor(denoise), baseLayer);
 
     // Scale with input / (1 - ratio) and set values to zero with
     // probability ratio.
-    mask = arma::randu<MatType>(denoise.n_rows, denoise.n_cols);
+    mask = arma::randu<arma::Mat<eT> >(denoise.n_rows, denoise.n_cols);
     mask.transform([&](double val) { return (val > ratio); });
 
-    baseLayer->Parameters() = denoise % mask;
-    baseLayer->Forward(input, output);
+    arma::mat tmp = denoise % mask;
+    boost::apply_visitor(ParametersSetVisitor(tmp), baseLayer);
+
+    boost::apply_visitor(ForwardVisitor(input, output), baseLayer);
 
     output = output * scale;
   }
 }
 
-template<typename MatType>
-void DropConnectType<MatType>::Backward(
-    const MatType& input,
-    const MatType& gy,
-    MatType& g)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void DropConnect<InputDataType, OutputDataType>::Backward(
+    const arma::Mat<eT>& input,
+    const arma::Mat<eT>& gy,
+    arma::Mat<eT>& g)
 {
-  baseLayer->Backward(input, gy, g);
+  boost::apply_visitor(BackwardVisitor(input, gy, g), baseLayer);
 }
 
-template<typename MatType>
-void DropConnectType<MatType>::Gradient(
-    const MatType& input,
-    const MatType& error,
-    MatType& gradient)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void DropConnect<InputDataType, OutputDataType>::Gradient(
+    const arma::Mat<eT>& input,
+    const arma::Mat<eT>& error,
+    arma::Mat<eT>& /* gradient */)
 {
-  baseLayer->Gradient(input, error, gradient);
+  boost::apply_visitor(GradientVisitor(input, error),
+      baseLayer);
 
   // Denoise the weights.
-  baseLayer->Parameters() = denoise;
+  boost::apply_visitor(ParametersSetVisitor(denoise), baseLayer);
 }
 
-template<typename MatType>
-void DropConnectType<MatType>::ComputeOutputDimensions()
-{
-  // Propagate input dimensions to the base layer.
-  baseLayer->InputDimensions() = this->inputDimensions;
-  this->outputDimensions = baseLayer->OutputDimensions();
-}
-
-template<typename MatType>
-void DropConnectType<MatType>::SetWeights(
-    typename MatType::elem_type* weightsPtr)
-{
-  baseLayer->SetWeights(weightsPtr);
-}
-
-template<typename MatType>
+template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
-void DropConnectType<MatType>::serialize(
+void DropConnect<InputDataType, OutputDataType>::serialize(
     Archive& ar, const uint32_t /* version */)
 {
-  ar(cereal::base_class<Layer<MatType>>(this));
+  // Delete the old network first, if needed.
+  if (cereal::is_loading<Archive>())
+  {
+    boost::apply_visitor(DeleteVisitor(), baseLayer);
+  }
 
   ar(CEREAL_NVP(ratio));
   ar(CEREAL_NVP(scale));
-  ar(CEREAL_POINTER(baseLayer));
+  ar(CEREAL_VARIANT_POINTER(baseLayer));
+
+  if (cereal::is_loading<Archive>())
+  {
+    network.clear();
+    network.push_back(baseLayer);
+  }
 }
 
 }  // namespace ann

src/mlpack/methods/ann/layer/dropout.hpp

@@ -15,16 +15,18 @@
 
 #include <mlpack/prereqs.hpp>
 
-#include "layer.hpp"
-
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
+
 /**
  * The dropout layer is a regularizer that randomly with probability 'ratio'
  * sets input values to zero and scales the remaining elements by factor 1 /
  * (1 - ratio) rather than during test time so as to keep the expected sum same.
- * When the layer is in testing mode, there is no change in the input.
+ * In the deterministic mode (during testing), there is no change in the input.
+ *
+ * Note: During training you should set deterministic to false and during
+ * testing you should set deterministic to true.
  *
  * For more information, see the following.
  *
@@ -41,11 +43,14 @@ namespace ann /** Artificial Neural Network. */ {
  * }
  * @endcode
  *
- * @tparam MatType Matrix representation to accept as input and use for
- *    computation.
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
  */
-template<typename MatType = arma::mat>
-class DropoutType : public Layer<MatType>
+template<typename InputDataType = arma::mat,
+         typename OutputDataType = arma::mat>
+class Dropout
 {
  public:
   /**
@@ -53,22 +58,19 @@ class DropoutType : public Layer<MatType>
    *
    * @param ratio The probability of setting a value to zero.
    */
-  DropoutType(const double ratio = 0.5);
+  Dropout(const double ratio = 0.5);
 
-  //! Clone the DropoutType object. This handles polymorphism correctly.
-  DropoutType* Clone() const { return new DropoutType(*this); }
+  //! Copy Constructor
+  Dropout(const Dropout& layer);
 
-  // Virtual destructor.
-  virtual ~DropoutType() { }
+  //! Move Constructor
+  Dropout(const Dropout&&);
 
-  //! Copy the given DropoutType.
-  DropoutType(const DropoutType& other);
-  //! Take ownership of the given DropoutType.
-  DropoutType(DropoutType&& other);
-  //! Copy the given DropoutType.
-  DropoutType& operator=(const DropoutType& other);
-  //! Take ownership of the given DropoutType.
-  DropoutType& operator=(DropoutType&& other);
+  //! Copy assignment operator
+  Dropout& operator=(const Dropout& layer);
+
+  //! Move assignment operator
+  Dropout& operator=(Dropout&& layer);
 
   /**
    * Ordinary feed forward pass of the dropout layer.
@@ -76,7 +78,8 @@ class DropoutType : public Layer<MatType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
-  void Forward(const MatType& input, MatType& output);
+  template<typename eT>
+  void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);
 
   /**
    * Ordinary feed backward pass of the dropout layer.
@@ -85,7 +88,25 @@ class DropoutType : public Layer<MatType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const MatType& /* input */, const MatType& gy, MatType& g);
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& /* input */,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g);
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the detla.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  //! The value of the deterministic parameter.
+  bool Deterministic() const { return deterministic; }
+  //! Modify the value of the deterministic parameter.
+  bool& Deterministic() { return deterministic; }
 
   //! The probability of setting a value to zero.
   double Ratio() const { return ratio; }
@@ -104,20 +125,24 @@ class DropoutType : public Layer<MatType>
   void serialize(Archive& ar, const uint32_t /* version */);
 
  private:
-  //! Locally-stored mask object.
-  MatType mask;
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
+  //! Locally-stored mast object.
+  OutputDataType mask;
 
   //! The probability of setting a value to zero.
   double ratio;
 
   //! The scale fraction.
   double scale;
-}; // class DropoutType
 
-// Convenience typedefs.
-
-// Standard Dropout layer.
-typedef DropoutType<arma::mat> Dropout;
+  //! If true dropout and scaling is disabled, see notes above.
+  bool deterministic;
+}; // class Dropout
 
 } // namespace ann
 } // namespace mlpack

src/mlpack/methods/ann/layer/dropout_impl.hpp

@@ -19,66 +19,73 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template<typename MatType>
-DropoutType<MatType>::DropoutType(
+template<typename InputDataType, typename OutputDataType>
+Dropout<InputDataType, OutputDataType>::Dropout(
     const double ratio) :
     ratio(ratio),
-    scale(1.0 / (1.0 - ratio))
+    scale(1.0 / (1.0 - ratio)),
+    deterministic(false)
 {
   // Nothing to do here.
 }
 
-template<typename MatType>
-DropoutType<MatType>::DropoutType(const DropoutType& other) :
-    Layer<MatType>(other),
-    ratio(other.ratio),
-    scale(other.scale)
+template<typename InputDataType, typename OutputDataType>
+Dropout<InputDataType, OutputDataType>::Dropout(
+    const Dropout& layer) :
+    ratio(layer.ratio),
+    scale(layer.scale),
+    deterministic(layer.deterministic)
 {
-  // Nothing to do.
+  // Nothing to do here.
 }
 
-template<typename MatType>
-DropoutType<MatType>::DropoutType(DropoutType&& other) :
-    Layer<MatType>(std::move(other)),
-    ratio(std::move(other.ratio)),
-    scale(std::move(other.scale))
+template<typename InputDataType, typename OutputDataType>
+Dropout<InputDataType, OutputDataType>::Dropout(
+    const Dropout&& layer) :
+    ratio(std::move(layer.ratio)),
+    scale(std::move(scale)),
+    deterministic(std::move(deterministic))
 {
-  // Nothing to do.
+  // Nothing to do here.
 }
 
-template<typename MatType>
-DropoutType<MatType>&
-DropoutType<MatType>::operator=(const DropoutType& other)
+template<typename InputDataType, typename OutputDataType>
+Dropout<InputDataType, OutputDataType>&
+Dropout<InputDataType, OutputDataType>::
+operator=(const Dropout& layer)
 {
-  if (&other != this)
+  if (this != &layer)
   {
-    Layer<MatType>::operator=(other);
-    ratio = other.ratio;
-    scale = other.scale;
+    ratio = layer.ratio;
+    scale = layer.scale;
+    deterministic = layer.deterministic;
   }
-
   return *this;
 }
 
-template<typename MatType>
-DropoutType<MatType>&
-DropoutType<MatType>::operator=(DropoutType&& other)
+template<typename InputDataType, typename OutputDataType>
+Dropout<InputDataType, OutputDataType>&
+Dropout<InputDataType, OutputDataType>::
+operator=(Dropout&& layer)
 {
-  if (&other != this)
+  if (this != &layer)
   {
-    Layer<MatType>::operator=(std::move(other));
-    ratio = std::move(other.ratio);
-    scale = std::move(other.scale);
+    ratio = std::move(layer.ratio);
+    scale = std::move(layer.scale);
+    deterministic = std::move(layer.deterministic);
   }
-
   return *this;
 }
 
-template<typename MatType>
-void DropoutType<MatType>::Forward(const MatType& input, MatType& output)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void Dropout<InputDataType, OutputDataType>::Forward(
+    const arma::Mat<eT>& input,
+    arma::Mat<eT>& output)
 {
-  // The dropout mask will not be multiplied in testing mode.
-  if (!this->training)
+  // The dropout mask will not be multiplied in the deterministic mode
+  // (during testing).
+  if (deterministic)
   {
     output = input;
   }
@@ -86,29 +93,28 @@ void DropoutType<MatType>::Forward(const MatType& input, MatType& output)
   {
     // Scale with input / (1 - ratio) and set values to zero with probability
     // 'ratio'.
-    mask = arma::randu<MatType>(input.n_rows, input.n_cols);
+    mask = arma::randu<arma::Mat<eT> >(input.n_rows, input.n_cols);
     mask.transform([&](double val) { return (val > ratio); });
     output = input % mask * scale;
   }
 }
 
-template<typename MatType>
-void DropoutType<MatType>::Backward(
-    const MatType& /* input */,
-    const MatType& gy,
-    MatType& g)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void Dropout<InputDataType, OutputDataType>::Backward(
+    const arma::Mat<eT>& /* input */,
+    const arma::Mat<eT>& gy,
+    arma::Mat<eT>& g)
 {
   g = gy % mask * scale;
 }
 
-template<typename MatType>
+template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
-void DropoutType<MatType>::serialize(
+void Dropout<InputDataType, OutputDataType>::serialize(
     Archive& ar,
     const uint32_t /* version */)
 {
-  ar(cereal::base_class<Layer<MatType>>(this));
-
   ar(CEREAL_NVP(ratio));
 
   // Reset scale.

src/mlpack/methods/ann/layer/elu.hpp

@@ -6,9 +6,9 @@
  * Definition of the ELU activation function as described by Djork-Arne Clevert,
  * Thomas Unterthiner and Sepp Hochreiter.
  *
- * Definition of the SELU function as introduced by Klambauer et. al. in Self
- * Neural Networks.  The SELU activation function keeps the mean and variance of
- * the input invariant.
+ * Definition of the SELU function as introduced by
+ * Klambauer et. al. in Self Neural Networks.  The SELU activation
+ * function keeps the mean and variance of the input invariant.
  *
  * In short, SELU = lambda * ELU, with 'alpha' and 'lambda' fixed for
  * normalized inputs.
@@ -26,8 +26,6 @@
 
 #include <mlpack/prereqs.hpp>
 
-#include "layer.hpp"
-
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
@@ -63,6 +61,7 @@ namespace ann /** Artificial Neural Network. */ {
  * }
  * @endcode
  *
+ *
  * The SELU activation function is defined by
  *
  * @f{eqnarray*}{
@@ -93,19 +92,23 @@ namespace ann /** Artificial Neural Network. */ {
  * }
  * @endcode
  *
- * In testing mode, there is no computation of the derivative.
+ * In the deterministic mode, there is no computation of the derivative.
  *
+ * @note During training deterministic should be set to false and during
+ *       testing/inference deterministic should be set to true.
  * @note Make sure to use SELU activation function with normalized inputs and
  *       weights initialized with Lecun Normal Initialization.
  *
- * @tparam InputType The type of the layer's inputs. The layer automatically
- *     cast inputs to this type (Default: arma::mat).
- * @tparam OutputType The type of the computation which also causes the output
- *     to also be in this type. The type also allows the computation and weight
- *     type to differ from the input type (Default: arma::mat).
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
  */
-template <typename InputType = arma::mat, typename OutputType = arma::mat>
-class ELUType : public Layer<InputType, OutputType>
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+class ELU
 {
  public:
   /**
@@ -113,7 +116,7 @@ class ELUType : public Layer<InputType, OutputType>
    *
    * NOTE: Use this constructor for SELU activation function.
    */
-  ELUType();
+  ELU();
 
   /**
    * Create the ELU object using the specified parameter. The non zero
@@ -123,10 +126,8 @@ class ELUType : public Layer<InputType, OutputType>
    * @note Use this constructor for ELU activation function.
    * @param alpha Scale parameter for the negative factor.
    */
-  ELUType(const double alpha);
+  ELU(const double alpha);
 
-  //! Clone the ELUType object. This handles polymorphism correctly.
-  ELUType* Clone() const { return new ELUType(*this); }
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function
    * f(x) by propagating the activity forward through f.
@@ -134,6 +135,7 @@ class ELUType : public Layer<InputType, OutputType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
+  template<typename InputType, typename OutputType>
   void Forward(const InputType& input, OutputType& output);
 
   /**
@@ -145,13 +147,29 @@ class ELUType : public Layer<InputType, OutputType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const InputType& input, const OutputType& gy, OutputType& g);
+  template<typename DataType>
+  void Backward(const DataType& input, const DataType& gy, DataType& g);
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
 
   //! Get the non zero gradient.
   double const& Alpha() const { return alpha; }
   //! Modify the non zero gradient.
   double& Alpha() { return alpha; }
 
+  //! Get the value of deterministic parameter.
+  bool Deterministic() const { return deterministic; }
+  //! Modify the value of deterministic parameter.
+  bool& Deterministic() { return deterministic; }
+
   //! Get the lambda parameter.
   double const& Lambda() const { return lambda; }
 
@@ -162,27 +180,31 @@ class ELUType : public Layer<InputType, OutputType>
   void serialize(Archive& ar, const uint32_t /* version */);
 
  private:
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
   //! Locally stored first derivative of the activation function.
-  OutputType derivative;
+  arma::mat derivative;
 
   //! ELU Hyperparameter (0 < alpha)
   //! SELU parameter fixed to 1.6732632423543774 for normalized inputs.
   double alpha;
 
-  //! Lambda parameter used for multiplication of ELU function.
+  //! Lambda Parameter used for multiplication of ELU function.
   //! For ELU activation function, lambda = 1.
   //! For SELU activation function, lambda = 1.0507009873554802 for normalized
   //! inputs.
   double lambda;
-}; // class ELUType
 
-// Convenience typedefs.
+  //! If true the derivative computation is disabled, see notes above.
+  bool deterministic;
+}; // class ELU
 
-// Standard flexible ReLU layer.
-typedef ELUType<arma::mat, arma::mat> ELU;
-
-// Standard ELU layer.
-typedef ELUType<arma::mat, arma::mat> SELU;
+// Template alias for SELU using ELU class.
+using SELU = ELU<arma::mat, arma::mat>;
 
 } // namespace ann
 } // namespace mlpack

src/mlpack/methods/ann/layer/elu_impl.hpp

@@ -26,59 +26,66 @@ namespace ann /** Artificial Neural Network. */ {
 
 // This constructor is called for SELU activation function.  The values of
 // alpha and lambda are constant for normalized inputs.
-template<typename InputType, typename OutputType>
-ELUType<InputType, OutputType>::ELUType() :
+template<typename InputDataType, typename OutputDataType>
+ELU<InputDataType, OutputDataType>::ELU() :
     alpha(1.6732632423543774),
-    lambda(1.0507009873554802)
+    lambda(1.0507009873554802),
+    deterministic(false)
 {
   // Nothing to do here.
 }
 
-// This constructor is called for ELU activation function. The value of lambda
-// is fixed and equal to 1. 'alpha' is a hyperparameter.
-template<typename InputType, typename OutputType>
-ELUType<InputType, OutputType>::ELUType(const double alpha) :
+// This constructor is called for ELU activation function.  The value of lambda
+// is fixed and equal to 1.  'alpha' is a hyperparameter.
+template<typename InputDataType, typename OutputDataType>
+ELU<InputDataType, OutputDataType>::ELU(const double alpha) :
     alpha(alpha),
-    lambda(1)
+    lambda(1),
+    deterministic(false)
 {
   // Nothing to do here.
 }
 
+template<typename InputDataType, typename OutputDataType>
 template<typename InputType, typename OutputType>
-void ELUType<InputType, OutputType>::Forward(
+void ELU<InputDataType, OutputDataType>::Forward(
     const InputType& input, OutputType& output)
 {
-  output.ones();
+  output = arma::ones<OutputDataType>(arma::size(input));
   for (size_t i = 0; i < input.n_elem; ++i)
   {
     if (input(i) < DBL_MAX)
     {
-      output(i) = (input(i) > 0) ? lambda * input(i) : lambda * alpha *
-          (std::exp(input(i)) - 1);
+      output(i) = (input(i) > 0) ? lambda * input(i) : lambda *
+          alpha * (std::exp(input(i)) - 1);
     }
   }
 
-  if (!deterministic)
-  {
-    for (size_t i = 0; i < input.n_elem; ++i)
-      derivative(i) = (input(i) > 0) ? lambda : output(i) + lambda * alpha;
-  }
+    if (!deterministic)
+    {
+      derivative.set_size(arma::size(input));
+      for (size_t i = 0; i < input.n_elem; ++i)
+      {
+        derivative(i) = (input(i) > 0) ? lambda : output(i) +
+            lambda * alpha;
+      }
+    }
 }
 
-template<typename InputType, typename OutputType>
-void ELUType<InputType, OutputType>::Backward(
-    const InputType& /* input */, const OutputType& gy, OutputType& g)
+template<typename InputDataType, typename OutputDataType>
+template<typename DataType>
+void ELU<InputDataType, OutputDataType>::Backward(
+    const DataType& /* input */, const DataType& gy, DataType& g)
 {
   g = gy % derivative;
 }
 
-template<typename InputType, typename OutputType>
+template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
-void ELUType<InputType, OutputType>::serialize(
-    Archive& ar, const uint32_t /* version */)
+void ELU<InputDataType, OutputDataType>::serialize(
+    Archive& ar,
+    const uint32_t /* version */)
 {
-  ar(cereal::base_class<Layer<InputType, OutputType>>(this));
-
   ar(CEREAL_NVP(alpha));
   ar(CEREAL_NVP(lambda));
 }

src/mlpack/methods/ann/layer/fast_lstm.hpp

@@ -1,4 +1,3 @@
-// Temporarily drop.
 /**
  * @file methods/ann/layer/fast_lstm.hpp
  * @author Marcus Edel
@@ -16,7 +15,6 @@
 
 #include <mlpack/prereqs.hpp>
 #include <limits>
-#include "layer.hpp"
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
@@ -56,36 +54,36 @@ namespace ann /** Artificial Neural Network. */ {
  *
  * \see LSTM for a standard implementation of the LSTM layer.
  *
- * @tparam InputType Type of the input data (arma::colvec, arma::mat,
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
- * @tparam OutputType Type of the output data (arma::colvec, arma::mat,
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
  */
 template <
-    typename InputType = arma::mat,
-    typename OutputType = arma::mat
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
 >
-class FastLSTMType : public Layer<InputType, OutputType>
+class FastLSTM
 {
  public:
   // Convenience typedefs.
-  typedef typename InputType::elem_type InputET;
-  typedef typename OutputType::elem_type OutputET;
+  typedef typename InputDataType::elem_type InputElemType;
+  typedef typename OutputDataType::elem_type ElemType;
 
-  //! Create the FastLSTMType object.
-  FastLSTMType();
+  //! Create the Fast LSTM object.
+  FastLSTM();
 
   //! Copy Constructor
-  FastLSTMType(const FastLSTMType& layer);
+  FastLSTM(const FastLSTM& layer);
 
   //! Move Constructor
-  FastLSTMType(FastLSTMType&& layer);
+  FastLSTM(FastLSTM&& layer);
 
   //! Copy assignment operator
-  FastLSTMType& operator=(const FastLSTMType& layer);
+  FastLSTM& operator=(const FastLSTM& layer);
 
   //! Move assignment operator
-  FastLSTMType& operator=(FastLSTMType&& layer);
+  FastLSTM& operator=(FastLSTM&& layer);
 
   /**
    * Create the Fast LSTM layer object using the specified parameters.
@@ -94,12 +92,9 @@ class FastLSTMType : public Layer<InputType, OutputType>
    * @param outSize The number of output units.
    * @param rho Maximum number of steps to backpropagate through time (BPTT).
    */
-  FastLSTMType(const size_t inSize,
-               const size_t outSize,
-               const size_t rho = std::numeric_limits<size_t>::max());
-
-  //! Clone the FastLSTMType object. This handles polymorphism correctly.
-  FastLSTMType* Clone() const { return new FastLSTMType(*this); }
+  FastLSTM(const size_t inSize,
+           const size_t outSize,
+           const size_t rho = std::numeric_limits<size_t>::max());
 
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function
@@ -108,6 +103,7 @@ class FastLSTMType : public Layer<InputType, OutputType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
+  template<typename InputType, typename OutputType>
   void Forward(const InputType& input, OutputType& output);
 
   /**
@@ -119,16 +115,17 @@ class FastLSTMType : public Layer<InputType, OutputType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
+  template<typename InputType, typename ErrorType, typename GradientType>
   void Backward(const InputType& input,
-                const OutputType& gy,
-                OutputType& g);
+                const ErrorType& gy,
+                GradientType& g);
 
-  /**
+  /*
    * Reset the layer parameter.
    */
   void Reset();
 
-  /**
+  /*
    * Resets the cell to accept a new input. This breaks the BPTT chain starts a
    * new one.
    *
@@ -136,16 +133,17 @@ class FastLSTMType : public Layer<InputType, OutputType>
    */
   void ResetCell(const size_t size);
 
-  /**
+  /*
    * Calculate the gradient using the output delta and the input activation.
    *
    * @param input The input parameter used for calculating the gradient.
    * @param error The calculated error.
    * @param gradient The calculated gradient.
    */
+  template<typename InputType, typename ErrorType, typename GradientType>
   void Gradient(const InputType& input,
-                const OutputType& error,
-                OutputType& gradient);
+                const ErrorType& error,
+                GradientType& gradient);
 
   //! Get the maximum number of steps to backpropagate through time (BPTT).
   size_t Rho() const { return rho; }
@@ -153,9 +151,24 @@ class FastLSTMType : public Layer<InputType, OutputType>
   size_t& Rho() { return rho; }
 
   //! Get the parameters.
-  OutputType const& Parameters() const { return weights; }
+  OutputDataType const& Parameters() const { return weights; }
   //! Modify the parameters.
-  OutputType& Parameters() { return weights; }
+  OutputDataType& Parameters() { return weights; }
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  //! Get the gradient.
+  OutputDataType const& Gradient() const { return grad; }
+  //! Modify the gradient.
+  OutputDataType& Gradient() { return grad; }
 
   //! Get the number of input units.
   size_t InSize() const { return inSize; }
@@ -169,15 +182,14 @@ class FastLSTMType : public Layer<InputType, OutputType>
     return 4 * outSize * inSize + 4 * outSize + 4 * outSize * outSize;
   }
 
-  const std::vector<size_t> OutputDimensions() const
+  //! Get the shape of the input.
+  size_t InputShape() const
   {
-    std::vector<size_t> result(inputDimensions.size(), 0);
-    result[0] = outSize;
-    return result;
+    return inSize;
   }
 
   /**
-   * Serialize the layer.
+   * Serialize the layer
    */
   template<typename Archive>
   void serialize(Archive& ar, const uint32_t /* version */);
@@ -189,6 +201,7 @@ class FastLSTMType : public Layer<InputType, OutputType>
    * @param input The input data.
    * @param sigmoid The matrix to store the sigmoid approximation into.
    */
+  template<typename InputType, typename OutputType>
   void FastSigmoid(const InputType& input, OutputType& sigmoids)
   {
     for (size_t i = 0; i < input.n_elem; ++i)
@@ -201,10 +214,10 @@ class FastLSTMType : public Layer<InputType, OutputType>
    * @param data The given data sample for the sigmoid approximation.
    * @tparam The sigmoid approximation.
    */
-  OutputET FastSigmoid(const InputET data)
+  ElemType FastSigmoid(const InputElemType data)
   {
-    OutputET x = 0.5 * data;
-    OutputET z;
+    ElemType x = 0.5 * data;
+    ElemType z;
     if (x >= 0)
     {
       if (x < 1.7)
@@ -216,7 +229,7 @@ class FastLSTMType : public Layer<InputType, OutputType>
     }
     else
     {
-      OutputET xx = -x;
+      ElemType xx = -x;
       if (xx < 1.7)
         z = -(1.5 * xx / (1 + xx));
       else if (xx < 3)
@@ -247,10 +260,10 @@ class FastLSTMType : public Layer<InputType, OutputType>
   size_t gradientStep;
 
   //! Locally-stored weight object.
-  OutputType weights;
+  OutputDataType weights;
 
   //! Locally-stored previous output.
-  OutputType prevOutput;
+  OutputDataType prevOutput;
 
   //! Locally-stored batch size.
   size_t batchSize;
@@ -263,50 +276,56 @@ class FastLSTMType : public Layer<InputType, OutputType>
   size_t gradientStepIdx;
 
   //! Locally-stored cell activation error.
-  OutputType cellActivationError;
+  OutputDataType cellActivationError;
+
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored gradient object.
+  OutputDataType grad;
 
   //! Locally-stored output parameter object.
-  OutputType outputParameter;
+  OutputDataType outputParameter;
 
   //! Weights between the output and gate.
-  OutputType output2GateWeight;
+  OutputDataType output2GateWeight;
 
   //! Weights between the input and gate.
-  OutputType input2GateWeight;
+  OutputDataType input2GateWeight;
 
   //! Bias between the input and gate.
-  OutputType input2GateBias;
+  OutputDataType input2GateBias;
 
   //! Locally-stored gate parameter.
-  OutputType gate;
+  OutputDataType gate;
 
   //! Locally-stored gate activation.
-  OutputType gateActivation;
+  OutputDataType gateActivation;
 
   //! Locally-stored state activation.
-  OutputType stateActivation;
+  OutputDataType stateActivation;
 
   //! Locally-stored cell parameter.
-  OutputType cell;
+  OutputDataType cell;
 
   //! Locally-stored cell activation error.
-  OutputType cellActivation;
+  OutputDataType cellActivation;
 
   //! Locally-stored foget gate error.
-  OutputType forgetGateError;
+  OutputDataType forgetGateError;
 
   //! Locally-stored previous error.
-  OutputType prevError;
+  OutputDataType prevError;
+
+  //! Locally-stored output parameters.
+  OutputDataType outParameter;
 
   //! Locally-stored current rho size.
   size_t rhoSize;
 
   //! Current backpropagate through time steps.
   size_t bpttSteps;
-}; // class FastLSTMType.
-
-// Standard FastLSTM layer.
-typedef FastLSTMType<arma::mat, arma::mat> FastLSTM;
+}; // class FastLSTM
 
 } // namespace ann
 } // namespace mlpack

src/mlpack/methods/ann/layer/fast_lstm_impl.hpp

@@ -19,14 +19,14 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template<typename InputType, typename OutputType>
-FastLSTMType<InputType, OutputType>::FastLSTMType()
+template<typename InputDataType, typename OutputDataType>
+FastLSTM<InputDataType, OutputDataType>::FastLSTM()
 {
   // Nothing to do here.
 }
 
-template<typename InputType, typename OutputType>
-FastLSTMType<InputType, OutputType>::FastLSTMType(
+template <typename InputDataType, typename OutputDataType>
+FastLSTM<InputDataType, OutputDataType>::FastLSTM(
     const size_t inSize, const size_t outSize, const size_t rho) :
     inSize(inSize),
     outSize(outSize),
@@ -45,8 +45,8 @@ FastLSTMType<InputType, OutputType>::FastLSTMType(
   weights.set_size(WeightSize(), 1);
 }
 
-template<typename InputType, typename OutputType>
-FastLSTMType<InputType, OutputType>::FastLSTMType(const FastLSTMType& layer) :
+template<typename InputDataType, typename OutputDataType>
+FastLSTM<InputDataType, OutputDataType>::FastLSTM(const FastLSTM& layer) :
     inSize(layer.inSize),
     outSize(layer.outSize),
     rho(layer.rho),
@@ -57,14 +57,15 @@ FastLSTMType<InputType, OutputType>::FastLSTMType(const FastLSTMType& layer) :
     batchSize(layer.batchSize),
     batchStep(layer.batchStep),
     gradientStepIdx(layer.gradientStepIdx),
+    grad(layer.grad),
     rhoSize(layer.rho),
     bpttSteps(layer.bpttSteps)
 {
   // Nothing to do here.
 }
 
-template<typename InputType, typename OutputType>
-FastLSTMType<InputType, OutputType>::FastLSTMType(FastLSTMType&& layer) :
+template<typename InputDataType, typename OutputDataType>
+FastLSTM<InputDataType, OutputDataType>::FastLSTM(FastLSTM&& layer) :
     inSize(std::move(layer.inSize)),
     outSize(std::move(layer.outSize)),
     rho(std::move(layer.rho)),
@@ -75,15 +76,16 @@ FastLSTMType<InputType, OutputType>::FastLSTMType(FastLSTMType&& layer) :
     batchSize(std::move(layer.batchSize)),
     batchStep(std::move(layer.batchStep)),
     gradientStepIdx(std::move(layer.gradientStepIdx)),
+    grad(std::move(layer.grad)),
     rhoSize(std::move(layer.rho)),
     bpttSteps(std::move(layer.bpttSteps))
 {
   // Nothing to do here.
 }
 
-template<typename InputType, typename OutputType>
-FastLSTMType<InputType, OutputType>&
-FastLSTMType<InputType, OutputType>::operator=(const FastLSTMType& layer)
+template<typename InputDataType, typename OutputDataType>
+FastLSTM<InputDataType, OutputDataType>&
+FastLSTM<InputDataType, OutputDataType>::operator=(const FastLSTM& layer)
 {
   if (this != &layer)
   {
@@ -97,15 +99,16 @@ FastLSTMType<InputType, OutputType>::operator=(const FastLSTMType& layer)
     batchSize = layer.batchSize;
     batchStep = layer.batchStep;
     gradientStepIdx = layer.gradientStepIdx;
+    grad = layer.grad;
     rhoSize = layer.rho;
     bpttSteps = layer.bpttSteps;
   }
   return *this;
 }
 
-template<typename InputType, typename OutputType>
-FastLSTMType<InputType, OutputType>&
-FastLSTMType<InputType, OutputType>::operator=(FastLSTMType&& layer)
+template<typename InputDataType, typename OutputDataType>
+FastLSTM<InputDataType, OutputDataType>&
+FastLSTM<InputDataType, OutputDataType>::operator=(FastLSTM&& layer)
 {
   if (this != &layer)
   {
@@ -119,30 +122,31 @@ FastLSTMType<InputType, OutputType>::operator=(FastLSTMType&& layer)
     batchSize = std::move(layer.batchSize);
     batchStep = std::move(layer.batchStep);
     gradientStepIdx = std::move(layer.gradientStepIdx);
+    grad = std::move(layer.grad);
     rhoSize = std::move(layer.rho);
     bpttSteps = std::move(layer.bpttSteps);
   }
   return *this;
 }
 
-template<typename InputType, typename OutputType>
-void FastLSTMType<InputType, OutputType>::Reset()
+template<typename InputDataType, typename OutputDataType>
+void FastLSTM<InputDataType, OutputDataType>::Reset()
 {
   // Set the weight parameter for the input to gate layer (linear layer) using
   // the overall layer parameter matrix.
-  input2GateWeight = OutputType(weights.memptr(),
+  input2GateWeight = OutputDataType(weights.memptr(),
       4 * outSize, inSize, false, false);
-  input2GateBias = OutputType(weights.memptr() + input2GateWeight.n_elem,
+  input2GateBias = OutputDataType(weights.memptr() + input2GateWeight.n_elem,
       4 * outSize, 1, false, false);
 
   // Set the weight parameter for the output to gate layer
   // (linear no bias layer) using the overall layer parameter matrix.
-  output2GateWeight = OutputType(weights.memptr() + input2GateWeight.n_elem
+  output2GateWeight = OutputDataType(weights.memptr() + input2GateWeight.n_elem
       + input2GateBias.n_elem, 4 * outSize, outSize, false, false);
 }
 
-template<typename InputType, typename OutputType>
-void FastLSTMType<InputType, OutputType>::ResetCell(const size_t size)
+template<typename InputDataType, typename OutputDataType>
+void FastLSTM<InputDataType, OutputDataType>::ResetCell(const size_t size)
 {
   if (size == std::numeric_limits<size_t>::max())
     return;
@@ -175,8 +179,9 @@ void FastLSTMType<InputType, OutputType>::ResetCell(const size_t size)
   outParameter.zeros(outSize, (size + 1) * batchSize);
 }
 
+template<typename InputDataType, typename OutputDataType>
 template<typename InputType, typename OutputType>
-void FastLSTMType<InputType, OutputType>::Forward(
+void FastLSTM<InputDataType, OutputDataType>::Forward(
     const InputType& input, OutputType& output)
 {
   // Check if the batch size changed, the number of cols is defines the input
@@ -193,8 +198,8 @@ void FastLSTMType<InputType, OutputType>::Forward(
       forwardStep, forwardStep + batchStep);
   gate.cols(forwardStep, forwardStep + batchStep).each_col() += input2GateBias;
 
-  InputType sigmoidOut(gateActivation.colptr(forwardStep),
-      gateActivation.n_rows, batchStep, false, false);
+  arma::subview<double> sigmoidOut = gateActivation.cols(forwardStep,
+      forwardStep + batchStep);
   FastSigmoid(
       gate.submat(0, forwardStep, 3 * outSize - 1, forwardStep + batchStep),
       sigmoidOut);
@@ -242,19 +247,20 @@ void FastLSTMType<InputType, OutputType>::Forward(
   }
 }
 
-template<typename InputType, typename OutputType>
-void FastLSTMType<InputType, OutputType>::Backward(
-  const InputType& /* input */, const OutputType& gy, OutputType& g)
+template<typename InputDataType, typename OutputDataType>
+template<typename InputType, typename ErrorType, typename GradientType>
+void FastLSTM<InputDataType, OutputDataType>::Backward(
+  const InputType& /* input */, const ErrorType& gy, GradientType& g)
 {
-  OutputType gyLocal;
+  ErrorType gyLocal;
   if (gradientStepIdx > 0)
   {
     gyLocal = gy + output2GateWeight.t() * prevError;
   }
   else
   {
-    gyLocal = OutputType(((OutputType&) gy).memptr(), gy.n_rows, gy.n_cols,
-        false, false);
+    gyLocal = ErrorType(((ErrorType&) gy).memptr(), gy.n_rows, gy.n_cols, false,
+        false);
   }
 
   cellActivationError = gyLocal % gateActivation.submat(outSize,
@@ -313,11 +319,12 @@ void FastLSTMType<InputType, OutputType>::Backward(
   }
 }
 
-template<typename InputType, typename OutputType>
-void FastLSTMType<InputType, OutputType>::Gradient(
+template<typename InputDataType, typename OutputDataType>
+template<typename InputType, typename ErrorType, typename GradientType>
+void FastLSTM<InputDataType, OutputDataType>::Gradient(
     const InputType& input,
-    const OutputType& /* error */,
-    OutputType& gradient)
+    const ErrorType& /* error */,
+    GradientType& gradient)
 {
   // Gradient of the input to gate layer.
   gradient.submat(0, 0, input2GateWeight.n_elem - 1, 0) =
@@ -341,13 +348,11 @@ void FastLSTMType<InputType, OutputType>::Gradient(
   }
 }
 
-template<typename InputType, typename OutputType>
+template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
-void FastLSTMType<InputType, OutputType>::serialize(
+void FastLSTM<InputDataType, OutputDataType>::serialize(
     Archive& ar, const uint32_t /* version */)
 {
-  ar(cereal::base_class<Layer<InputType, OutputType>>(this));
-
   ar(CEREAL_NVP(weights));
   ar(CEREAL_NVP(inSize));
   ar(CEREAL_NVP(outSize));
@@ -366,10 +371,7 @@ void FastLSTMType<InputType, OutputType>::serialize(
   ar(CEREAL_NVP(cellActivation));
   ar(CEREAL_NVP(forgetGateError));
   ar(CEREAL_NVP(prevError));
-
-  // Restore aliases.
-  if (Archive::is_loading::value)
-    Reset();
+  ar(CEREAL_NVP(outParameter));
 }
 
 } // namespace ann

src/mlpack/methods/ann/layer/flexible_relu.hpp

@@ -3,9 +3,10 @@
  * @author Aarush Gupta
  * @author Manthan-R-Sheth
  *
- * Definition of the FlexibleReLU layer as described by Suo Qiu, Xiangmin Xu and
- * Bolun Cai in "FReLU: Flexible Rectified Linear Units for Improving
- * Convolutional Neural Networks".
+ * Definition of FlexibleReLU layer as described by
+ * Suo Qiu, Xiangmin Xu and Bolun Cai in
+ * "FReLU: Flexible Rectified Linear Units for Improving Convolutional
+ * Neural Networks", 2018
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -17,8 +18,6 @@
 
 #include <mlpack/prereqs.hpp>
 
-#include "layer.hpp"
-
 namespace mlpack {
 namespace ann /**Artificial Neural Network*/ {
 
@@ -28,10 +27,10 @@ namespace ann /**Artificial Neural Network*/ {
  * @f{eqnarray*}{
  * f(x) &=& \max(0,x)+alpha \\
  * f'(x) &=& \left\{
- *   \begin{array}{lr}
- *      1 & : x > 0 \\
- *      0 & : x \le 0
- *   \end{array}
+ * 	 \begin{array}{lr}
+ * 	   1 & : x > 0 \\
+ * 	   0 & : x \le 0
+ * 	 \end{array}
  * \right.
  * @f}
  *
@@ -48,33 +47,34 @@ namespace ann /**Artificial Neural Network*/ {
  * }
  * @endcode
  *
- * @tparam InputType The type of the layer's inputs. The layer automatically
- *     cast inputs to this type (Default: arma::mat).
- * @tparam OutputType The type of the computation which also causes the output
- *     to also be in this type. The type also allows the computation and weight
- *     type to differ from the input type (Default: arma::mat).
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mar,
+ *         arma::sp_mat or arma::cube)
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube)
  */
-template<typename InputType = arma::mat, typename OutputType = arma::mat>
-class FlexibleReLUType : public Layer<InputType, OutputType>
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+class FlexibleReLU
 {
  public:
   /**
-   * Create the FlexibleReLU object using the specified alpha parameter.
-   * The trainable alpha parameter controls the range of the ReLU function.
-   * (Default alpha = 0).
    *
-   * @param alpha Parameter to adjust the range of the ReLU function.
+   * Create the FlexibleReLU object using the specified parameters.
+   * The non zero parameter can be adjusted by specifying the parameter
+   * alpha which controls the range of the relu function. (Default alpha = 0)
+   * This parameter is trainable.
+   *
+   * @param alpha Parameter for adjusting the range of the relu function.
+   *
    */
-  FlexibleReLUType(const double alpha = 0);
-
-  //! Clone the FlexibleReLUType object. This handles polymorphism correctly.
-  FlexibleReLUType* Clone() const { return new FlexibleReLUType(*this); }
+  FlexibleReLU(const double alpha = 0);
 
   /**
-   * Reset the layer parameter (alpha). The method is called to
-   * assign the allocated memory to the learnable layer parameter.
+   * Reset the layer parameter.
    */
-  void SetWeights(typename OutputType::elem_type* weightsPtr);
+  void Reset();
 
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function
@@ -83,6 +83,7 @@ class FlexibleReLUType : public Layer<InputType, OutputType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
+  template<typename InputType, typename OutputType>
   void Forward(const InputType& input, OutputType& output);
 
   /**
@@ -94,7 +95,8 @@ class FlexibleReLUType : public Layer<InputType, OutputType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const InputType& input, const OutputType& gy, OutputType& g);
+  template<typename DataType>
+  void Backward(const DataType& input, const DataType& gy, DataType& g);
 
   /**
    * Calculate the gradient using the output delta and the input activation.
@@ -103,22 +105,36 @@ class FlexibleReLUType : public Layer<InputType, OutputType>
    * @param error The calculated error.
    * @param gradient The calculated gradient.
    */
-  void Gradient(const InputType& input,
-                const OutputType& error,
-                OutputType& gradient);
+  template<typename eT>
+  void Gradient(const arma::Mat<eT>& input,
+                const arma::Mat<eT>& error,
+                arma::Mat<eT>& gradient);
 
   //! Get the parameters.
-  OutputType const& Parameters() const { return alpha; }
+  OutputDataType const& Parameters() const { return alpha; }
   //! Modify the parameters.
-  OutputType& Parameters() { return alpha; }
+  OutputDataType& Parameters() { return alpha; }
 
-  //! Get the parameter controlling the range of the ReLU function.
-  const double& Alpha() const { return alpha; }
-  //! Modify the parameter controlling the range of the ReLU function.
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta;}
+
+  //! Get the gradient.
+  OutputDataType const& Gradient() const { return gradient; }
+  //! Modify the gradient.
+  OutputDataType& Gradient() { return gradient; }
+
+  //! Get the parameter controlling the range of the relu function.
+  double const& Alpha() const { return alpha; }
+  //! Modify the parameter controlling the range of the relu function.
   double& Alpha() { return alpha; }
 
-  const size_t WeightSize() const { return 1; }
-
   /**
    * Serialize the layer.
    */
@@ -126,20 +142,21 @@ class FlexibleReLUType : public Layer<InputType, OutputType>
   void serialize(Archive& ar, const uint32_t /* version*/);
 
  private:
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
   //! Parameter object.
-  OutputType alpha;
+  OutputDataType alpha;
 
-  //! Parameter controlling the range of the ReLU function.
+  //! Locally-stored gradient object.
+  OutputDataType gradient;
+
+  //! Parameter controlling the range of the rectifier function
   double userAlpha;
-
-  //! Whether or not a forward pass has ever been performed.
-  bool initialized;
-}; // class FlexibleReLUType
-
-// Convenience typedefs.
-
-// Standard flexible ReLU layer.
-typedef FlexibleReLUType<arma::mat, arma::mat> FlexibleReLU;
+}; // class FlexibleReLU
 
 } // namespace ann
 } // namespace mlpack

src/mlpack/methods/ann/layer/flexible_relu_impl.hpp

@@ -18,67 +18,66 @@
 #define MLPACK_METHODS_ANN_LAYER_FLEXIBLERELU_IMPL_HPP
 
 #include "flexible_relu.hpp"
+#include<algorithm>
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template<typename InputType, typename OutputType>
-FlexibleReLUType<InputType, OutputType>::FlexibleReLUType(const double alpha) :
-    userAlpha(alpha),
-    initialized(false)
+template<typename InputDataType, typename OutputDataType>
+FlexibleReLU<InputDataType, OutputDataType>::FlexibleReLU(
+    const double alpha) : userAlpha(alpha)
 {
   this->alpha.set_size(1, 1);
-  this->alpha(0) = alpha;
+  this->alpha(0) = userAlpha;
 }
 
-template<typename InputType, typename OutputType>
-void FlexibleReLUType<InputType, OutputType>::SetWeights(
-    typename OutputType::elem_type* weightsPtr)
+template<typename InputDataType, typename OutputDataType>
+void FlexibleReLU<InputDataType, OutputDataType>::Reset()
 {
-  alpha = OutputType(weightsPtr, 1, 1, false, false);
+  //! Set value of alpha to the one given by user.
+  alpha(0) = userAlpha;
 }
 
+template<typename InputDataType, typename OutputDataType>
 template<typename InputType, typename OutputType>
-void FlexibleReLUType<InputType, OutputType>::Forward(
+void FlexibleReLU<InputDataType, OutputDataType>::Forward(
     const InputType& input, OutputType& output)
 {
-  if (!initialized)
-  {
-    alpha[0] = userAlpha;
-    initialized = true;
-  }
-
   output = arma::clamp(input, 0.0, DBL_MAX) + alpha(0);
 }
 
-template<typename InputType, typename OutputType>
-void FlexibleReLUType<InputType, OutputType>::Backward(
-    const InputType& input, const OutputType& gy, OutputType& g)
+template<typename InputDataType, typename OutputDataType>
+template<typename DataType>
+void FlexibleReLU<InputDataType, OutputDataType>::Backward(
+    const DataType& input, const DataType& gy, DataType& g)
 {
-  // Compute the first derivative of FlexibleReLU function.
+  //! Compute the first derivative of FlexibleReLU function.
   g = gy % arma::clamp(arma::sign(input), 0.0, 1.0);
 }
 
-template<typename InputType, typename OutputType>
-void FlexibleReLUType<InputType, OutputType>::Gradient(
-    const InputType& input,
-    const OutputType& error,
-    OutputType& gradient)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void FlexibleReLU<InputDataType, OutputDataType>::Gradient(
+    const arma::Mat<eT>& input,
+    const arma::Mat<eT>& error,
+    arma::Mat<eT>& gradient)
 {
+  if (gradient.n_elem == 0)
+  {
+    gradient.set_size(1, 1);
+  }
+
   gradient(0) = arma::accu(error) / input.n_cols;
 }
 
-template<typename InputType, typename OutputType>
+
+template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
-void FlexibleReLUType<InputType, OutputType>::serialize(
+void FlexibleReLU<InputDataType, OutputDataType>::serialize(
     Archive& ar,
     const uint32_t /* version*/)
 {
-  ar(cereal::base_class<Layer<InputType, OutputType>>(this));
-
   ar(CEREAL_NVP(alpha));
-  ar(CEREAL_NVP(userAlpha));
-  ar(CEREAL_NVP(initialized));
 }
 
 } // namespace ann

src/mlpack/methods/ann/layer/glimpse.hpp

@@ -1,4 +1,3 @@
-// Temporarily drop.
 /**
  * @file methods/ann/layer/glimpse.hpp
  * @author Marcus Edel
@@ -77,16 +76,16 @@ class MeanPoolingRule
  * (down-scaled cropped images) of increasing scale around a given location in a
  * given image.
  *
- * @tparam InputType Type of the input data (arma::colvec, arma::mat,
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
- * @tparam OutputType Type of the output data (arma::colvec, arma::mat,
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
  */
 template <
-    typename InputType = arma::mat,
-    typename OutputType = arma::mat
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
 >
-class GlimpseType : public Layer<InputType, OutputType>
+class Glimpse
 {
  public:
   /**
@@ -101,12 +100,12 @@ class GlimpseType : public Layer<InputType, OutputType>
    * @param inputWidth The input width of the given input data.
    * @param inputHeight The input height of the given input data.
    */
-  GlimpseType(const size_t inSize = 0,
-              const size_t size = 0,
-              const size_t depth = 3,
-              const size_t scale = 2,
-              const size_t inputWidth = 0,
-              const size_t inputHeight = 0);
+  Glimpse(const size_t inSize = 0,
+          const size_t size = 0,
+          const size_t depth = 3,
+          const size_t scale = 2,
+          const size_t inputWidth = 0,
+          const size_t inputHeight = 0);
 
   /**
    * Ordinary feed forward pass of the glimpse layer.
@@ -114,7 +113,8 @@ class GlimpseType : public Layer<InputType, OutputType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
-  void Forward(const InputType& input, OutputType& output);
+  template<typename eT>
+  void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);
 
   /**
    * Ordinary feed backward pass of the glimpse layer.
@@ -123,13 +123,27 @@ class GlimpseType : public Layer<InputType, OutputType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const InputType& /* input */,
-                const OutputType& gy,
-                OutputType& g);
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& /* input */,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g);
+
+  //! Get the output parameter.
+  OutputDataType& OutputParameter() const {return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the detla.
+  OutputDataType& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
 
   //! Set the locationthe x and y coordinate of the center of the output
   //! glimpse.
-  void Location(const arma::mat& location) { this->location = location; }
+  void Location(const arma::mat& location)
+  {
+    this->location = location;
+  }
 
   //! Get the input width.
   size_t const& InputWidth() const { return inputWidth; }
@@ -151,6 +165,11 @@ class GlimpseType : public Layer<InputType, OutputType>
   //! Modify the output height.
   size_t& OutputHeight() { return outputHeight; }
 
+  //! Get the value of the deterministic parameter.
+  bool Deterministic() const { return deterministic; }
+  //! Modify the value of the deterministic parameter.
+  bool& Deterministic() { return deterministic; }
+
   //! Get the number of patches to crop per glimpse.
   size_t const& Depth() const { return depth; }
 
@@ -163,14 +182,10 @@ class GlimpseType : public Layer<InputType, OutputType>
   //! Get the used glimpse size (height = width).
   size_t GlimpseSize() const { return size;}
 
-  const std::vector<size_t> OutputDimensions() const
+  //! Get the shape of the input.
+  size_t InputShape() const
   {
-    std::vector<size_t> result(inputDimensions.size(), 0);
-    result[0] = outputWidth;
-    result[1] = outputHeight;
-    for (size_t i = 2; i < inputDimensions.size(); ++i)
-      result[i] = inputDimensions[i];
-    return result;
+    return inSize;
   }
 
   /**
@@ -180,7 +195,7 @@ class GlimpseType : public Layer<InputType, OutputType>
   void serialize(Archive& ar, const uint32_t /* version */);
 
  private:
-  /**
+  /*
    * Transform the given input by changing rows to columns.
    *
    * @param w The input matrix used to perform the transformation.
@@ -220,9 +235,10 @@ class GlimpseType : public Layer<InputType, OutputType>
    * @param input The input to be apply the pooling rule.
    * @param output The pooled result.
    */
+  template<typename eT>
   void Pooling(const size_t kSize,
-               const InputType& input,
-               OutputType& output)
+               const arma::Mat<eT>& input,
+               arma::Mat<eT>& output)
   {
     const size_t rStep = kSize;
     const size_t cStep = kSize;
@@ -244,20 +260,21 @@ class GlimpseType : public Layer<InputType, OutputType>
    * @param error The error used to perform the unpooling operation.
    * @param output The pooled result.
    */
-  void Unpooling(const InputType& input,
-                 const OutputType& error,
-                 OutputType& output)
+  template<typename eT>
+  void Unpooling(const arma::Mat<eT>& input,
+                 const arma::Mat<eT>& error,
+                 arma::Mat<eT>& output)
   {
     const size_t rStep = input.n_rows / error.n_rows;
     const size_t cStep = input.n_cols / error.n_cols;
 
-    OutputType unpooledError;
+    arma::Mat<eT> unpooledError;
     for (size_t j = 0; j < input.n_cols; j += cStep)
     {
       for (size_t i = 0; i < input.n_rows; i += rStep)
       {
-        const InputType& inputArea = input(arma::span(i, i + rStep - 1),
-                                           arma::span(j, j + cStep - 1));
+        const arma::Mat<eT>& inputArea = input(arma::span(i, i + rStep - 1),
+                                               arma::span(j, j + cStep - 1));
 
         pooling.Unpooling(inputArea, error(i / rStep, j / cStep),
             unpooledError);
@@ -275,7 +292,8 @@ class GlimpseType : public Layer<InputType, OutputType>
    * @param input The input to be apply the ReSampling rule.
    * @param output The pooled result.
    */
-  void ReSampling(const InputType& input, OutputType& output)
+  template<typename eT>
+  void ReSampling(const arma::Mat<eT>& input, arma::Mat<eT>& output)
   {
     double wRatio = (double) (input.n_rows - 1) / (size - 1);
     double hRatio = (double) (input.n_cols - 1) / (size - 1);
@@ -319,9 +337,10 @@ class GlimpseType : public Layer<InputType, OutputType>
    * @param error The error used to perform the DownwardReSampling operation.
    * @param output The DownwardReSampled result.
    */
-  void DownwardReSampling(const InputType& input,
-                          const OutputType& error,
-                          OutputType& output)
+  template<typename eT>
+  void DownwardReSampling(const arma::Mat<eT>& input,
+                          const arma::Mat<eT>& error,
+                          arma::Mat<eT>& output)
   {
     double iWidth = input.n_rows - 1;
     double iHeight = input.n_cols - 1;
@@ -385,30 +404,36 @@ class GlimpseType : public Layer<InputType, OutputType>
   //! Locally-stored output height.
   size_t outputHeight;
 
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
   //! Locally-stored depth of the input.
   size_t inputDepth;
 
   //! Locally-stored transformed input parameter.
-  arma::Cube<typename InputType::elem_type> inputTemp;
+  arma::cube inputTemp;
 
   //! Locally-stored transformed output parameter.
-  arma::Cube<typename OutputType::elem_type> outputTemp;
+  arma::cube outputTemp;
 
   //! The x and y coordinate of the center of the output glimpse.
-  OutputType location;
+  arma::mat location;
 
   //! Locally-stored object to perform the mean pooling operation.
   MeanPoolingRule pooling;
 
   //! Location-stored module location parameter.
-  std::vector<OutputType> locationParameter;
+  std::vector<arma::mat> locationParameter;
 
   //! Location-stored transformed gradient paramter.
-  arma::Cube<typename OutputType::elem_type> gTemp;
-}; // class GlimpseType
+  arma::cube gTemp;
 
-// Standard Glimpse layer.
-typedef GlimpseType<arma::mat, arma::mat> Glimpse;
+  //! If true use maximum a posteriori during the forward pass.
+  bool deterministic;
+}; // class GlimpseLayer
 
 } // namespace ann
 } // namespace mlpack

src/mlpack/methods/ann/layer/glimpse_impl.hpp

@@ -20,8 +20,8 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template <typename InputType, typename OutputType>
-GlimpseType<InputType, OutputType>::GlimpseType(
+template <typename InputDataType, typename OutputDataType>
+Glimpse<InputDataType, OutputDataType>::Glimpse(
     const size_t inSize,
     const size_t size,
     const size_t depth,
@@ -42,14 +42,13 @@ GlimpseType<InputType, OutputType>::GlimpseType(
   // Nothing to do here.
 }
 
-template <typename InputType, typename OutputType>
-void GlimpseType<InputType, OutputType>::Forward(
-    const InputType& input, OutputType& output)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void Glimpse<InputDataType, OutputDataType>::Forward(
+    const arma::Mat<eT>& input, arma::Mat<eT>& output)
 {
-  inputTemp = arma::Cube<typename InputType::elem_type>(input.colptr(0),
-      inputWidth, inputHeight, inSize);
-  outputTemp = arma::Cube<typename OutputType::elem_type>(size, size, depth *
-      inputTemp.n_slices);
+  inputTemp = arma::cube(input.colptr(0), inputWidth, inputHeight, inSize);
+  outputTemp = arma::Cube<eT>(size, size, depth * inputTemp.n_slices);
 
   location = input.submat(0, 1, 1, 1);
 
@@ -67,8 +66,7 @@ void GlimpseType<InputType, OutputType>::Forward(
     {
       size_t padSize = std::floor((glimpseSize - 1) / 2);
 
-      arma::Cube<typename InputType::elem_type> inputPadded =
-          arma::zeros<arma::Cube<typename InputType::elem_type>>(
+      arma::Cube<eT> inputPadded = arma::zeros<arma::Cube<eT> >(
           inputTemp.n_rows + padSize * 2, inputTemp.n_cols + padSize * 2,
           inputTemp.n_slices / inSize);
 
@@ -100,8 +98,9 @@ void GlimpseType<InputType, OutputType>::Forward(
         for (size_t j = (inputIdx + depthIdx * (depth - 1)), paddedSlice = 0;
             j < outputTemp.n_slices; j += (inSize * depth), paddedSlice++)
         {
-          InputType poolingInput = inputPadded.subcube(x, y, paddedSlice,
-              x + glimpseSize - 1, y + glimpseSize - 1, paddedSlice);
+          arma::Mat<eT> poolingInput = inputPadded.subcube(x, y,
+              paddedSlice, x + glimpseSize - 1, y + glimpseSize - 1,
+              paddedSlice);
 
           if (scale == 2)
           {
@@ -121,19 +120,19 @@ void GlimpseType<InputType, OutputType>::Forward(
     outputTemp.slice(i) = arma::trans(outputTemp.slice(i));
   }
 
-  output = OutputType(outputTemp.memptr(), outputTemp.n_elem, 1);
+  output = arma::Mat<eT>(outputTemp.memptr(), outputTemp.n_elem, 1);
 
   outputWidth = outputTemp.n_rows;
   outputHeight = outputTemp.n_cols;
 }
 
-template <typename InputType, typename OutputType>
-void GlimpseType<InputType, OutputType>::Backward(
-    const InputType& /* input */, const OutputType& gy, OutputType& g)
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void Glimpse<InputDataType, OutputDataType>::Backward(
+    const arma::Mat<eT>& /* input */, const arma::Mat<eT>& gy, arma::Mat<eT>& g)
 {
   // Generate a cube using the backpropagated error matrix.
-  arma::Cube<typename OutputType::elem_type> mappedError =
-      arma::zeros<arma::Cube<typename OutputType::elem_type>>(outputWidth,
+  arma::Cube<eT> mappedError = arma::zeros<arma::cube>(outputWidth,
       outputHeight, 1);
 
   location = locationParameter.back();
@@ -143,13 +142,13 @@ void GlimpseType<InputType, OutputType>::Backward(
   {
     for (size_t i = 0; i < gy.n_cols; ++i)
     {
-      mappedError.slice(s + i) = OutputType(gy.memptr(),
+      mappedError.slice(s + i) = arma::Mat<eT>(gy.memptr(),
           outputWidth, outputHeight);
     }
   }
 
-  gTemp = arma::zeros<arma::Cube<typename InputType::elem_type>>(
-      inputTemp.n_rows, inputTemp.n_cols, inputTemp.n_slices);
+  gTemp = arma::zeros<arma::cube>(inputTemp.n_rows, inputTemp.n_cols,
+      inputTemp.n_slices);
 
   for (size_t inputIdx = 0; inputIdx < inSize; inputIdx++)
   {
@@ -158,8 +157,7 @@ void GlimpseType<InputType, OutputType>::Backward(
     {
       size_t padSize = std::floor((glimpseSize - 1) / 2);
 
-      arma::Cube<typename InputType::elem_type> inputPadded =
-          arma::zeros<arma::Cube<typename InputType::elem_type>>(
+      arma::Cube<eT> inputPadded = arma::zeros<arma::Cube<eT> >(
           inputTemp.n_rows + padSize * 2, inputTemp.n_cols +
           padSize * 2, inputTemp.n_slices / inSize);
 
@@ -186,8 +184,9 @@ void GlimpseType<InputType, OutputType>::Backward(
         for (size_t j = (inputIdx + depthIdx * (depth - 1)), paddedSlice = 0;
             j < mappedError.n_slices; j += (inSize * depth), paddedSlice++)
         {
-          OutputType poolingOutput = inputPadded.subcube(x, y, paddedSlice,
-              x + glimpseSize - 1, y + glimpseSize - 1, paddedSlice);
+          arma::Mat<eT> poolingOutput = inputPadded.subcube(x, y,
+               paddedSlice, x + glimpseSize - 1, y + glimpseSize - 1,
+               paddedSlice);
 
           if (scale == 2)
           {
@@ -212,16 +211,14 @@ void GlimpseType<InputType, OutputType>::Backward(
   }
 
   Transform(gTemp);
-  g = OutputType(gTemp.memptr(), gTemp.n_elem, 1);
+  g = arma::mat(gTemp.memptr(), gTemp.n_elem, 1);
 }
 
-template <typename InputType, typename OutputType>
+template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
-void GlimpseType<InputType, OutputType>::serialize(
+void Glimpse<InputDataType, OutputDataType>::serialize(
     Archive& ar, const uint32_t /* version */)
 {
-  ar(cereal::base_class<Layer<InputType, OutputType>>(this));
-
   ar(CEREAL_NVP(inSize));
   ar(CEREAL_NVP(size));
   ar(CEREAL_NVP(depth));

src/mlpack/methods/ann/layer/gru.hpp

@@ -1,4 +1,3 @@
-// Temporarily drop.
 /**
  * @file methods/ann/layer/gru.hpp
  * @author Sumedh Ghaisas
@@ -32,6 +31,9 @@
 
 #include <mlpack/prereqs.hpp>
 
+#include "../visitor/delta_visitor.hpp"
+#include "../visitor/output_parameter_visitor.hpp"
+
 #include "layer_types.hpp"
 #include "add_merge.hpp"
 #include "sequential.hpp"
@@ -44,16 +46,16 @@ namespace ann /** Artificial Neural Network. */ {
  *
  * This cell can be used in RNN networks.
  *
- * @tparam InputType Type of the input data (arma::colvec, arma::mat,
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
- * @tparam OutputType Type of the output data (arma::colvec, arma::mat,
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
  */
 template <
-    typename InputType = arma::mat,
-    typename OutputType = arma::mat
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
 >
-class GRU : public Layer<InputType, OutputType>
+class GRU
 {
  public:
   //! Create the GRU object.
@@ -77,7 +79,8 @@ class GRU : public Layer<InputType, OutputType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
-  void Forward(const InputType& input, OutputType& output);
+  template<typename eT>
+  void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);
 
   /**
    * Ordinary feed backward pass of a neural network, calculating the function
@@ -88,9 +91,10 @@ class GRU : public Layer<InputType, OutputType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const InputType& /* input */,
-                const OutputType& gy,
-                OutputType& g);
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& /* input */,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g);
 
   /*
    * Calculate the gradient using the output delta and the input activation.
@@ -99,9 +103,10 @@ class GRU : public Layer<InputType, OutputType>
    * @param error The calculated error.
    * @param gradient The calculated gradient.
    */
-  void Gradient(const InputType& input,
-                const OutputType& /* error */,
-                OutputType& /* gradient */);
+  template<typename eT>
+  void Gradient(const arma::Mat<eT>& input,
+                const arma::Mat<eT>& /* error */,
+                arma::Mat<eT>& /* gradient */);
 
   /*
    * Resets the cell to accept a new input. This breaks the BPTT chain starts a
@@ -111,18 +116,38 @@ class GRU : public Layer<InputType, OutputType>
    */
   void ResetCell(const size_t size);
 
+  //! The value of the deterministic parameter.
+  bool Deterministic() const { return deterministic; }
+  //! Modify the value of the deterministic parameter.
+  bool& Deterministic() { return deterministic; }
+
   //! Get the maximum number of steps to backpropagate through time (BPTT).
   size_t Rho() const { return rho; }
   //! Modify the maximum number of steps to backpropagate through time (BPTT).
   size_t& Rho() { return rho; }
 
   //! Get the parameters.
-  OutputType const& Parameters() const { return weights; }
+  OutputDataType const& Parameters() const { return weights; }
   //! Modify the parameters.
-  OutputType& Parameters() { return weights; }
+  OutputDataType& Parameters() { return weights; }
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  //! Get the gradient.
+  OutputDataType const& Gradient() const { return gradient; }
+  //! Modify the gradient.
+  OutputDataType& Gradient() { return gradient; }
 
   //! Get the model modules.
-  std::vector<Layer<InputType, OutputType>*>& Model() { return network; }
+  std::vector<LayerTypes<> >& Model() { return network; }
 
   //! Get the number of input units.
   size_t InSize() const { return inSize; }
@@ -156,28 +181,37 @@ class GRU : public Layer<InputType, OutputType>
   size_t batchSize;
 
   //! Locally-stored weight object.
-  OutputType weights;
+  OutputDataType weights;
 
   //! Locally-stored input 2 gate module.
-  Layer<InputType, OutputType>* input2GateModule;
+  LayerTypes<> input2GateModule;
 
   //! Locally-stored output 2 gate module.
-  Layer<InputType, OutputType>* output2GateModule;
+  LayerTypes<> output2GateModule;
 
   //! Locally-stored output hidden state 2 gate module.
-  Layer<InputType, OutputType>* outputHidden2GateModule;
+  LayerTypes<> outputHidden2GateModule;
 
   //! Locally-stored input gate module.
-  Layer<InputType, OutputType>* inputGateModule;
+  LayerTypes<> inputGateModule;
 
   //! Locally-stored hidden state module.
-  Layer<InputType, OutputType>* hiddenStateModule;
+  LayerTypes<> hiddenStateModule;
 
   //! Locally-stored forget gate module.
-  Layer<InputType, OutputType>* forgetGateModule;
+  LayerTypes<> forgetGateModule;
+
+  //! Locally-stored output parameter visitor.
+  OutputParameterVisitor outputParameterVisitor;
+
+  //! Locally-stored delta visitor.
+  DeltaVisitor deltaVisitor;
+
+  //! Locally-stored delete visitor.
+  DeleteVisitor deleteVisitor;
 
   //! Locally-stored list of network modules.
-  std::vector<Layer<InputType, OutputType>*> network;
+  std::vector<LayerTypes<> > network;
 
   //! Locally-stored number of forward steps.
   size_t forwardStep;
@@ -189,34 +223,34 @@ class GRU : public Layer<InputType, OutputType>
   size_t gradientStep;
 
   //! Locally-stored output parameters.
-  std::list<OutputType> outParameter;
+  std::list<arma::mat> outParameter;
 
   //! Matrix of all zeroes to initialize the output
-  OutputType allZeros;
+  arma::mat allZeros;
 
   //! Iterator pointed to the last output produced by the cell
-  typename std::list<OutputType>::iterator prevOutput;
+  std::list<arma::mat>::iterator prevOutput;
 
   //! Iterator pointed to the last output processed by backward
-  typename std::list<OutputType>::iterator backIterator;
+  std::list<arma::mat>::iterator backIterator;
 
   //! Iterator pointed to the last output processed by gradient
-  typename std::list<OutputType>::iterator gradIterator;
+  std::list<arma::mat>::iterator gradIterator;
 
   //! Locally-stored previous error.
-  OutputType prevError;
+  arma::mat prevError;
 
   //! If true dropout and scaling is disabled, see notes above.
   bool deterministic;
 
   //! Locally-stored delta object.
-  OutputType delta;
+  OutputDataType delta;
 
   //! Locally-stored gradient object.
-  OutputType gradient;
+  OutputDataType gradient;
 
   //! Locally-stored output parameter object.
-  OutputType outputParameter;
+  OutputDataType outputParameter;
 }; // class GRU
 
 } // namespace ann

src/mlpack/methods/ann/layer/gru_impl.hpp

@@ -0,0 +1,411 @@
+/**
+ * @file methods/ann/layer/gru_impl.hpp
+ * @author Sumedh Ghaisas
+ *
+ * Implementation of the GRU class, which implements a gru network
+ * layer.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_GRU_IMPL_HPP
+#define MLPACK_METHODS_ANN_LAYER_GRU_IMPL_HPP
+
+// In case it hasn't yet been included.
+#include "gru.hpp"
+
+#include "../visitor/forward_visitor.hpp"
+#include "../visitor/backward_visitor.hpp"
+#include "../visitor/gradient_visitor.hpp"
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+template<typename InputDataType, typename OutputDataType>
+GRU<InputDataType, OutputDataType>::GRU()
+{
+  // Nothing to do here.
+}
+
+template <typename InputDataType, typename OutputDataType>
+GRU<InputDataType, OutputDataType>::GRU(
+    const size_t inSize,
+    const size_t outSize,
+    const size_t rho) :
+    inSize(inSize),
+    outSize(outSize),
+    rho(rho),
+    batchSize(1),
+    forwardStep(0),
+    backwardStep(0),
+    gradientStep(0),
+    deterministic(false)
+{
+  // Input specific linear layers(for zt, rt, ot).
+  input2GateModule = new Linear<>(inSize, 3 * outSize);
+
+  // Previous output gates (for zt and rt).
+  output2GateModule = new LinearNoBias<>(outSize, 2 * outSize);
+
+  // Previous output gate for ot.
+  outputHidden2GateModule = new LinearNoBias<>(outSize, outSize);
+
+  network.push_back(input2GateModule);
+  network.push_back(output2GateModule);
+  network.push_back(outputHidden2GateModule);
+
+  inputGateModule = new SigmoidLayer<>();
+  forgetGateModule = new SigmoidLayer<>();
+  hiddenStateModule = new TanHLayer<>();
+
+  network.push_back(inputGateModule);
+  network.push_back(hiddenStateModule);
+  network.push_back(forgetGateModule);
+
+  prevError = arma::zeros<arma::mat>(3 * outSize, batchSize);
+
+  allZeros = arma::zeros<arma::mat>(outSize, batchSize);
+
+  outParameter.emplace_back(allZeros.memptr(),
+      allZeros.n_rows, allZeros.n_cols, false, true);
+
+  prevOutput = outParameter.begin();
+  backIterator = outParameter.end();
+  gradIterator = outParameter.end();
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void GRU<InputDataType, OutputDataType>::Forward(
+    const arma::Mat<eT>& input, arma::Mat<eT>& output)
+{
+  if (input.n_cols != batchSize)
+  {
+    batchSize = input.n_cols;
+    prevError.resize(3 * outSize, batchSize);
+    allZeros.zeros(outSize, batchSize);
+    // Batch size better not change during an iteration...
+    if (outParameter.size() > 1)
+    {
+      Log::Fatal << "GRU<>::Forward(): batch size cannot change during a "
+          << "forward pass!" << std::endl;
+    }
+
+    outParameter.clear();
+    outParameter.emplace_back(allZeros.memptr(),
+        allZeros.n_rows, allZeros.n_cols, false, true);
+
+    prevOutput = outParameter.begin();
+    backIterator = outParameter.end();
+    gradIterator = outParameter.end();
+  }
+
+  // Process the input linearly(zt, rt, ot).
+  boost::apply_visitor(ForwardVisitor(input,
+      boost::apply_visitor(outputParameterVisitor, input2GateModule)),
+      input2GateModule);
+
+  // Process the output(zt, rt) linearly.
+  boost::apply_visitor(ForwardVisitor(*prevOutput,
+      boost::apply_visitor(outputParameterVisitor, output2GateModule)),
+      output2GateModule);
+
+  // Merge the outputs(zt and rt).
+  output = (boost::apply_visitor(outputParameterVisitor,
+      input2GateModule).submat(0, 0, 2 * outSize - 1, batchSize - 1) +
+      boost::apply_visitor(outputParameterVisitor, output2GateModule));
+
+  // Pass the first outSize through inputGate(it).
+  boost::apply_visitor(ForwardVisitor(output.submat(
+      0, 0, 1 * outSize - 1, batchSize - 1), boost::apply_visitor(
+      outputParameterVisitor, inputGateModule)), inputGateModule);
+
+  // Pass the second through forgetGate.
+  boost::apply_visitor(ForwardVisitor(output.submat(
+      1 * outSize, 0, 2 * outSize - 1, batchSize - 1),
+      boost::apply_visitor(outputParameterVisitor, forgetGateModule)),
+      forgetGateModule);
+
+  arma::mat modInput = (boost::apply_visitor(outputParameterVisitor,
+      forgetGateModule) % *prevOutput);
+
+  // Pass that through the outputHidden2GateModule.
+  boost::apply_visitor(ForwardVisitor(modInput,
+      boost::apply_visitor(outputParameterVisitor, outputHidden2GateModule)),
+      outputHidden2GateModule);
+
+  // Merge for ot.
+  arma::mat outputH = boost::apply_visitor(outputParameterVisitor,
+      input2GateModule).submat(2 * outSize, 0, 3 * outSize - 1, batchSize - 1) +
+      boost::apply_visitor(outputParameterVisitor, outputHidden2GateModule);
+
+  // Pass it through hiddenGate.
+  boost::apply_visitor(ForwardVisitor(outputH,
+      boost::apply_visitor(outputParameterVisitor, hiddenStateModule)),
+      hiddenStateModule);
+
+  // Update the output (nextOutput): cmul1 + cmul2
+  // Where cmul1 is input gate * prevOutput and
+  // cmul2 is (1 - input gate) * hidden gate.
+  output = (boost::apply_visitor(outputParameterVisitor, inputGateModule)
+      % (*prevOutput - boost::apply_visitor(outputParameterVisitor,
+      hiddenStateModule))) + boost::apply_visitor(outputParameterVisitor,
+      hiddenStateModule);
+
+  forwardStep++;
+  if (forwardStep == rho)
+  {
+    forwardStep = 0;
+    if (!deterministic)
+    {
+      outParameter.emplace_back(allZeros.memptr(),
+          allZeros.n_rows, allZeros.n_cols, false, true);
+      prevOutput = --outParameter.end();
+    }
+    else
+    {
+      *prevOutput = arma::mat(allZeros.memptr(),
+          allZeros.n_rows, allZeros.n_cols, false, true);
+    }
+  }
+  else if (!deterministic)
+  {
+    outParameter.push_back(output);
+    prevOutput = --outParameter.end();
+  }
+  else
+  {
+    if (forwardStep == 1)
+    {
+      outParameter.clear();
+      outParameter.push_back(output);
+
+      prevOutput = outParameter.begin();
+    }
+    else
+    {
+      *prevOutput = output;
+    }
+  }
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void GRU<InputDataType, OutputDataType>::Backward(
+  const arma::Mat<eT>& input, const arma::Mat<eT>& gy, arma::Mat<eT>& g)
+{
+  if (input.n_cols != batchSize)
+  {
+    batchSize = input.n_cols;
+    prevError.resize(3 * outSize, batchSize);
+    allZeros.zeros(outSize, batchSize);
+    // Batch size better not change during an iteration...
+    if (outParameter.size() > 1)
+    {
+      Log::Fatal << "GRU<>::Forward(): batch size cannot change during a "
+          << "forward pass!" << std::endl;
+    }
+
+    outParameter.clear();
+    outParameter.emplace_back(allZeros.memptr(),
+        allZeros.n_rows, allZeros.n_cols, false, true);
+
+    prevOutput = outParameter.begin();
+    backIterator = outParameter.end();
+    gradIterator = outParameter.end();
+  }
+
+  arma::Mat<eT> gyLocal;
+  if ((outParameter.size() - backwardStep  - 1) % rho != 0 && backwardStep != 0)
+  {
+    gyLocal = gy + boost::apply_visitor(deltaVisitor, output2GateModule);
+  }
+  else
+  {
+    gyLocal = arma::Mat<eT>(((arma::Mat<eT>&) gy).memptr(), gy.n_rows,
+        gy.n_cols, false, false);
+  }
+
+  if (backIterator == outParameter.end())
+  {
+    backIterator = --(--outParameter.end());
+  }
+
+  // Delta zt.
+  arma::mat dZt = gyLocal % (*backIterator -
+      boost::apply_visitor(outputParameterVisitor,
+      hiddenStateModule));
+
+  // Delta ot.
+  arma::mat dOt = gyLocal % (arma::ones<arma::mat>(outSize, batchSize) -
+      boost::apply_visitor(outputParameterVisitor, inputGateModule));
+
+  // Delta of input gate.
+  boost::apply_visitor(BackwardVisitor(boost::apply_visitor(
+      outputParameterVisitor, inputGateModule), dZt,
+      boost::apply_visitor(deltaVisitor, inputGateModule)),
+      inputGateModule);
+
+  // Delta of hidden gate.
+  boost::apply_visitor(BackwardVisitor(boost::apply_visitor(
+      outputParameterVisitor, hiddenStateModule), dOt,
+      boost::apply_visitor(deltaVisitor, hiddenStateModule)),
+      hiddenStateModule);
+
+  // Delta of outputHidden2GateModule.
+  boost::apply_visitor(BackwardVisitor(boost::apply_visitor(
+      outputParameterVisitor, outputHidden2GateModule),
+      boost::apply_visitor(deltaVisitor, hiddenStateModule),
+      boost::apply_visitor(deltaVisitor, outputHidden2GateModule)),
+      outputHidden2GateModule);
+
+  // Delta rt.
+  arma::mat dRt = boost::apply_visitor(deltaVisitor, outputHidden2GateModule) %
+      *backIterator;
+
+  // Delta of forget gate.
+  boost::apply_visitor(BackwardVisitor(boost::apply_visitor(
+      outputParameterVisitor, forgetGateModule), dRt,
+      boost::apply_visitor(deltaVisitor, forgetGateModule)),
+      forgetGateModule);
+
+  // Put delta zt.
+  prevError.submat(0, 0, 1 * outSize - 1, batchSize - 1) = boost::apply_visitor(
+      deltaVisitor, inputGateModule);
+
+  // Put delta rt.
+  prevError.submat(1 * outSize, 0, 2 * outSize - 1, batchSize - 1) =
+      boost::apply_visitor(deltaVisitor, forgetGateModule);
+
+  // Put delta ot.
+  prevError.submat(2 * outSize, 0, 3 * outSize - 1, batchSize - 1) =
+      boost::apply_visitor(deltaVisitor, hiddenStateModule);
+
+  // Get delta ht - 1 for input gate and forget gate.
+  arma::mat prevErrorSubview = prevError.submat(0, 0, 2 * outSize - 1,
+      batchSize - 1);
+  boost::apply_visitor(BackwardVisitor(boost::apply_visitor(
+      outputParameterVisitor, input2GateModule),
+      prevErrorSubview,
+      boost::apply_visitor(deltaVisitor, output2GateModule)),
+      output2GateModule);
+
+  // Add delta ht - 1 from hidden state.
+  boost::apply_visitor(deltaVisitor, output2GateModule) +=
+      boost::apply_visitor(deltaVisitor, outputHidden2GateModule) %
+      boost::apply_visitor(outputParameterVisitor, forgetGateModule);
+
+  // Add delta ht - 1 from ht.
+  boost::apply_visitor(deltaVisitor, output2GateModule) += gyLocal %
+      boost::apply_visitor(outputParameterVisitor, inputGateModule);
+
+  // Get delta input.
+  boost::apply_visitor(BackwardVisitor(boost::apply_visitor(
+      outputParameterVisitor, input2GateModule), prevError,
+      boost::apply_visitor(deltaVisitor, input2GateModule)),
+      input2GateModule);
+
+  backwardStep++;
+  backIterator--;
+
+  g = boost::apply_visitor(deltaVisitor, input2GateModule);
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void GRU<InputDataType, OutputDataType>::Gradient(
+    const arma::Mat<eT>& input,
+    const arma::Mat<eT>& /* error */,
+    arma::Mat<eT>& /* gradient */)
+{
+  if (input.n_cols != batchSize)
+  {
+    batchSize = input.n_cols;
+    prevError.resize(3 * outSize, batchSize);
+    allZeros.zeros(outSize, batchSize);
+    // Batch size better not change during an iteration...
+    if (outParameter.size() > 1)
+    {
+      Log::Fatal << "GRU<>::Forward(): batch size cannot change during a "
+          << "forward pass!" << std::endl;
+    }
+
+    outParameter.clear();
+    outParameter.emplace_back(allZeros.memptr(),
+        allZeros.n_rows, allZeros.n_cols, false, true);
+
+    prevOutput = outParameter.begin();
+    backIterator = outParameter.end();
+    gradIterator = outParameter.end();
+  }
+
+  if (gradIterator == outParameter.end())
+  {
+    gradIterator = --(--outParameter.end());
+  }
+
+  boost::apply_visitor(GradientVisitor(input, prevError), input2GateModule);
+
+  boost::apply_visitor(GradientVisitor(
+      *gradIterator,
+      prevError.submat(0, 0, 2 * outSize - 1, batchSize - 1)),
+      output2GateModule);
+
+  boost::apply_visitor(GradientVisitor(
+      *gradIterator % boost::apply_visitor(outputParameterVisitor,
+      forgetGateModule),
+      prevError.submat(2 * outSize, 0, 3 * outSize - 1, batchSize - 1)),
+      outputHidden2GateModule);
+
+  gradIterator--;
+}
+
+template<typename InputDataType, typename OutputDataType>
+void GRU<InputDataType, OutputDataType>::ResetCell(const size_t /* size */)
+{
+  outParameter.clear();
+  outParameter.emplace_back(allZeros.memptr(),
+    allZeros.n_rows, allZeros.n_cols, false, true);
+
+  prevOutput = outParameter.begin();
+  backIterator = outParameter.end();
+  gradIterator = outParameter.end();
+
+  forwardStep = 0;
+  backwardStep = 0;
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename Archive>
+void GRU<InputDataType, OutputDataType>::serialize(
+    Archive& ar, const uint32_t /* version */)
+{
+  // If necessary, clean memory from the old model.
+  if (cereal::is_loading<Archive>())
+  {
+    boost::apply_visitor(deleteVisitor, input2GateModule);
+    boost::apply_visitor(deleteVisitor, output2GateModule);
+    boost::apply_visitor(deleteVisitor, outputHidden2GateModule);
+    boost::apply_visitor(deleteVisitor, inputGateModule);
+    boost::apply_visitor(deleteVisitor, forgetGateModule);
+    boost::apply_visitor(deleteVisitor, hiddenStateModule);
+  }
+
+  ar(CEREAL_NVP(inSize));
+  ar(CEREAL_NVP(outSize));
+  ar(CEREAL_NVP(rho));
+
+  ar(CEREAL_VARIANT_POINTER(input2GateModule));
+  ar(CEREAL_VARIANT_POINTER(output2GateModule));
+  ar(CEREAL_VARIANT_POINTER(outputHidden2GateModule));
+  ar(CEREAL_VARIANT_POINTER(inputGateModule));
+  ar(CEREAL_VARIANT_POINTER(forgetGateModule));
+  ar(CEREAL_VARIANT_POINTER(hiddenStateModule));
+}
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/layer/hard_tanh.hpp

@@ -14,8 +14,6 @@
 
 #include <mlpack/prereqs.hpp>
 
-#include "layer.hpp"
-
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
@@ -39,14 +37,16 @@ namespace ann /** Artificial Neural Network. */ {
  * \right.
  * @f}
  *
- * @tparam InputType The type of the layer's inputs. The layer automatically
- *    cast inputs to this type (Default: arma::mat).
- * @tparam OutputType The type of the computation which also causes the output
- *    to also be in this type. The type also allows the computation and weight
- *    type to differ from the input type (Default: arma::mat).
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
  */
-template <typename InputType = arma::mat, typename OutputType = arma::mat>
-class HardTanHType : public Layer<InputType, OutputType>
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+class HardTanH
 {
  public:
   /**
@@ -57,10 +57,7 @@ class HardTanHType : public Layer<InputType, OutputType>
    * @param maxValue Range of the linear region maximum value.
    * @param minValue Range of the linear region minimum value.
    */
-  HardTanHType(const double maxValue = 1, const double minValue = -1);
-
-  //! Clone the HardTanHType object. This handles polymorphism correctly.
-  HardTanHType* Clone() const { return new HardTanHType(*this); }
+  HardTanH(const double maxValue = 1, const double minValue = -1);
 
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function
@@ -69,6 +66,7 @@ class HardTanHType : public Layer<InputType, OutputType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
+  template<typename InputType, typename OutputType>
   void Forward(const InputType& input, OutputType& output);
 
   /**
@@ -80,7 +78,20 @@ class HardTanHType : public Layer<InputType, OutputType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const InputType& input, const OutputType& gy, OutputType& g);
+  template<typename DataType>
+  void Backward(const DataType& input,
+                const DataType& gy,
+                DataType& g);
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
 
   //! Get the maximum value.
   double const& MaxValue() const { return maxValue; }
@@ -99,17 +110,18 @@ class HardTanHType : public Layer<InputType, OutputType>
   void serialize(Archive& ar, const uint32_t /* version */);
 
  private:
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
   //! Maximum value for the HardTanH function.
   double maxValue;
 
   //! Minimum value for the HardTanH function.
   double minValue;
-}; // class HardTanHType
-
-// Convenience typedefs.
-
-// Standard HardTanH layer.
-typedef HardTanHType<arma::mat, arma::mat> HardTanH;
+}; // class HardTanH
 
 } // namespace ann
 } // namespace mlpack

src/mlpack/methods/ann/layer/hard_tanh_impl.hpp

@@ -18,8 +18,8 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-template<typename InputType, typename OutputType>
-HardTanHType<InputType, OutputType>::HardTanHType(
+template<typename InputDataType, typename OutputDataType>
+HardTanH<InputDataType, OutputDataType>::HardTanH(
     const double maxValue,
     const double minValue) :
     maxValue(maxValue),
@@ -28,10 +28,12 @@ HardTanHType<InputType, OutputType>::HardTanHType(
   // Nothing to do here.
 }
 
+template<typename InputDataType, typename OutputDataType>
 template<typename InputType, typename OutputType>
-void HardTanHType<InputType, OutputType>::Forward(
+void HardTanH<InputDataType, OutputDataType>::Forward(
     const InputType& input, OutputType& output)
 {
+  output = input;
   for (size_t i = 0; i < input.n_elem; ++i)
   {
     output(i) = (output(i) > maxValue ? maxValue :
@@ -39,9 +41,10 @@ void HardTanHType<InputType, OutputType>::Forward(
   }
 }
 
-template<typename InputType, typename OutputType>
-void HardTanHType<InputType, OutputType>::Backward(
-    const InputType& input, const OutputType& gy, OutputType& g)
+template<typename InputDataType, typename OutputDataType>
+template<typename DataType>
+void HardTanH<InputDataType, OutputDataType>::Backward(
+    const DataType& input, const DataType& gy, DataType& g)
 {
   g = gy;
   for (size_t i = 0; i < input.n_elem; ++i)
@@ -53,14 +56,12 @@ void HardTanHType<InputType, OutputType>::Backward(
   }
 }
 
-template<typename InputType, typename OutputType>
+template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
-void HardTanHType<InputType, OutputType>::serialize(
+void HardTanH<InputDataType, OutputDataType>::serialize(
     Archive& ar,
     const uint32_t /* version */)
 {
-  ar(cereal::base_class<Layer<InputType, OutputType>>(this));
-
   ar(CEREAL_NVP(maxValue));
   ar(CEREAL_NVP(minValue));
 }

src/mlpack/methods/ann/layer/hardshrink.hpp

@@ -17,8 +17,6 @@
 
 #include <mlpack/prereqs.hpp>
 
-#include "layer.hpp"
-
 namespace mlpack {
 namespace ann /** Artifical Neural Network. */ {
 
@@ -39,28 +37,23 @@ namespace ann /** Artifical Neural Network. */ {
  * \f}
  *
  * \f$\lambda\f$ is set to 0.5 by default.
- *
- * @tparam InputType The type of the layer's inputs. The layer automatically
- *     cast inputs to this type (Default: arma::mat).
- * @tparam OutputType The type of the computation which also causes the output
- *     to also be in this type. The type also allows the computation and weight
- *     type to differ from the input type (Default: arma::mat).
  */
-template<typename InputType = arma::mat, typename OutputType = arma::mat>
-class HardShrinkType : public Layer<InputType, OutputType>
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+class HardShrink
 {
  public:
   /**
    * Create HardShrink object using specified hyperparameter lambda.
    *
-   * @param lambda Is calculated by multiplying the noise level sigma of the
-   *     input(noisy image) and a coefficient 'a' which is one of the training
-   *     parameters. Default value of lambda is 0.5.
+   * @param lambda Is calculated by multiplying the
+   * 		    noise level sigma of the input(noisy image) and a
+   * 		    coefficient 'a' which is one of the training parameters.
+   * 		    Default value of lambda is 0.5.
    */
-  HardShrinkType(const double lambda = 0.5);
-
-  //! Clone the HardShrinkType object. This handles polymorphism correctly.
-  HardShrinkType* Clone() const { return new HardShrinkType(*this); }
+  HardShrink(const double lambda = 0.5);
 
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function
@@ -69,6 +62,7 @@ class HardShrinkType : public Layer<InputType, OutputType>
    * @param input Input data used for evaluating the Hard Shrink function.
    * @param output Resulting output activation.
    */
+  template<typename InputType, typename OutputType>
   void Forward(const InputType& input, OutputType& output);
 
   /**
@@ -80,26 +74,42 @@ class HardShrinkType : public Layer<InputType, OutputType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const InputType& input, const OutputType& gy, OutputType& g);
+  template<typename DataType>
+  void Backward(const DataType& input,
+                DataType& gy,
+                DataType& g);
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
 
   //! Get the hyperparameter lambda.
   double const& Lambda() const { return lambda; }
   //! Modify the hyperparameter lambda.
   double& Lambda() { return lambda; }
 
-  //! Serialize the layer.
+  /**
+   * Serialize the layer.
+   */
   template<typename Archive>
   void serialize(Archive& ar, const uint32_t /* version */);
 
  private:
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
   //! Locally-stored hyperparameter lambda.
   double lambda;
-}; // class HardShrinkType
-
-// Convenience typedefs.
-
-// Standard HardShrink layer.
-typedef HardShrinkType<arma::mat, arma::mat> HardShrink;
+}; // class HardShrink
 
 } // namespace ann
 } // namespace mlpack

src/mlpack/methods/ann/layer/hardshrink_impl.hpp

@@ -20,35 +20,37 @@ namespace ann /** Artificial Neural Network. */ {
 
 // This constructor is called for Hard Shrink activation function.
 // 'lambda' is a hyperparameter.
-template<typename InputType, typename OutputType>
-HardShrinkType<InputType, OutputType>::HardShrinkType(const double lambda) :
+template<typename InputDataType, typename OutputDataType>
+HardShrink<InputDataType, OutputDataType>::HardShrink(const double lambda) :
     lambda(lambda)
 {
   // Nothing to do here.
 }
 
+template<typename InputDataType, typename OutputDataType>
 template<typename InputType, typename OutputType>
-void HardShrinkType<InputType, OutputType>::Forward(
+void HardShrink<InputDataType, OutputDataType>::Forward(
     const InputType& input, OutputType& output)
 {
   output = ((input > lambda) + (input < -lambda)) % input;
 }
 
-template<typename InputType, typename OutputType>
-void HardShrinkType<InputType, OutputType>::Backward(
-    const InputType& input, const OutputType& gy, OutputType& g)
+template<typename InputDataType, typename OutputDataType>
+template<typename DataType>
+void HardShrink<InputDataType, OutputDataType>::Backward(
+    const DataType& input, DataType& gy, DataType& g)
 {
-  g = gy % (arma::ones<OutputType>(arma::size(input)) - (input == 0));
+  DataType derivative;
+  derivative = (arma::ones(arma::size(input)) - (input == 0));
+  g = gy % derivative;
 }
 
-template<typename InputType, typename OutputType>
+template<typename InputDataType, typename OutputDataType>
 template<typename Archive>
-void HardShrinkType<InputType, OutputType>::serialize(
+void HardShrink<InputDataType, OutputDataType>::serialize(
     Archive& ar,
     const uint32_t /* version */)
 {
-  ar(cereal::base_class<Layer<InputType, OutputType>>(this));
-
   ar(CEREAL_NVP(lambda));
 }
 

src/mlpack/methods/ann/layer/highway.hpp

@@ -0,0 +1,270 @@
+/**
+ * @file methods/ann/layer/highway.hpp
+ * @author Konstantin Sidorov
+ * @author Saksham Bansal
+ *
+ * Definition of the Highway layer.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_HIGHWAY_HPP
+#define MLPACK_METHODS_ANN_LAYER_HIGHWAY_HPP
+
+#include <mlpack/prereqs.hpp>
+
+#include "../visitor/delete_visitor.hpp"
+#include "../visitor/delta_visitor.hpp"
+#include "../visitor/output_height_visitor.hpp"
+#include "../visitor/output_parameter_visitor.hpp"
+#include "../visitor/output_width_visitor.hpp"
+
+#include "layer_types.hpp"
+#include "add_merge.hpp"
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * Implementation of the Highway layer. The Highway class can vary its behavior
+ * between that of feed-forward fully connected network container and that
+ * of a layer which simply passes its inputs through depending on the transform
+ * gate. Note that the size of the input and output matrices of this class
+ * should be equal.
+ *
+ * For more information, refer the following paper.
+ *
+ * @code
+ * @article{Srivastava2015,
+ *   author  = {Rupesh Kumar Srivastava, Klaus Greff, Jurgen Schmidhuber},
+ *   title   = {Training Very Deep Networks},
+ *   journal = {Advances in Neural Information Processing Systems},
+ *   year    = {2015},
+ *   url     = {https://arxiv.org/abs/1507.06228},
+ * }
+ * @endcode
+ *
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ */
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat,
+    typename... CustomLayers>
+class Highway
+{
+ public:
+  //! Create the Highway object.
+  Highway();
+
+  /**
+   * Create the Highway object.
+   *
+   * @param inSize The number of input units.
+   * @param model Expose all the network modules.
+   */
+  Highway(const size_t inSize, const bool model = true);
+
+  //! Destroy the Highway object.
+  ~Highway();
+
+  /**
+   * Reset the layer parameter.
+   */
+  void Reset();
+
+  /**
+   * Ordinary feed-forward pass of a neural network, evaluating the function
+   * f(x) by propagating the activity forward through f.
+   *
+   * @param input Input data used for evaluating the specified function.
+   * @param output Resulting output activation.
+   */
+  template<typename eT>
+  void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);
+
+  /**
+   * Ordinary feed-backward pass of a neural network, calculating the function
+   * f(x) by propagating x backwards through f. Using the results from the
+   * feed-forward pass.
+   *
+   * @param * (input) The propagated input activation.
+   * @param gy The backpropagated error.
+   * @param g The calculated gradient.
+   */
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& /* input */,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g);
+
+  /**
+   * Calculate the gradient using the output delta and the input activation.
+   *
+   * @param input The input parameter used for calculating the gradient.
+   * @param error The calculated error.
+   * @param gradient The calculated gradient.
+   */
+  template<typename eT>
+  void Gradient(const arma::Mat<eT>& input,
+                const arma::Mat<eT>& error,
+                arma::Mat<eT>& gradient);
+
+  /**
+   * Add a new module to the model.
+   *
+   * @param args The layer parameter.
+   */
+  template <class LayerType, class... Args>
+  void Add(Args... args)
+  {
+    network.push_back(new LayerType(args...));
+    networkOwnerships.push_back(true);
+  }
+
+  /**
+   * Add a new module to the model.
+   *
+   * @param layer The Layer to be added to the model.
+   */
+  void Add(LayerTypes<CustomLayers...> layer)
+  {
+    network.push_back(layer);
+    networkOwnerships.push_back(false);
+  }
+
+  //! Return the modules of the model.
+  std::vector<LayerTypes<CustomLayers...> >& Model()
+  {
+    if (model)
+    {
+      return network;
+    }
+
+    return empty;
+  }
+
+  //! Get the parameters.
+  OutputDataType const& Parameters() const { return weights; }
+  //! Modify the parameters.
+  OutputDataType& Parameters() { return weights; }
+
+  //! Get the input parameter.
+  InputDataType const& InputParameter() const { return inputParameter; }
+  //! Modify the input parameter.
+  InputDataType& InputParameter() { return inputParameter; }
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  //! Get the gradient.
+  OutputDataType const& Gradient() const { return gradient; }
+  //! Modify the gradient.
+  OutputDataType& Gradient() { return gradient; }
+
+  //! Get the number of input units.
+  size_t InSize() const { return inSize; }
+
+  //! Get the shape of the input.
+  size_t InputShape() const
+  {
+    return inSize;
+  }
+
+  /**
+   * Serialize the layer.
+   */
+  template<typename Archive>
+  void serialize(Archive& ar, const uint32_t /* version */);
+
+ private:
+  //! Locally-stored number of input units.
+  size_t inSize;
+
+  //! Parameter which indicates if the modules should be exposed.
+  bool model;
+
+  //! Indicator if we already initialized the model.
+  bool reset;
+
+  //! Locally-stored network modules.
+  std::vector<LayerTypes<CustomLayers...> > network;
+
+  //! The list of network modules we are responsible for.
+  std::vector<bool> networkOwnerships;
+
+  //! Locally-stored empty list of modules.
+  std::vector<LayerTypes<CustomLayers...> > empty;
+
+  //! Locally-stored weight object.
+  OutputDataType weights;
+
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored gradient object.
+  OutputDataType gradient;
+
+  //! Weights for transformation of output.
+  OutputDataType transformWeight;
+
+  //! Bias for transformation of output.
+  OutputDataType transformBias;
+
+  //! Locally-stored transform gate parameters.
+  OutputDataType transformGate;
+
+  //! Locally-stored transform gate activation.
+  OutputDataType transformGateActivation;
+
+  //! Locally-stored transform gate error.
+  OutputDataType transformGateError;
+
+  //! Locally-stored input parameter object.
+  InputDataType inputParameter;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
+  //! The input width.
+  size_t width;
+
+  //! The input height.
+  size_t height;
+
+  //! The normal output without highway network.
+  OutputDataType networkOutput;
+
+  //! Locally-stored delta visitor.
+  DeltaVisitor deltaVisitor;
+
+  //! Locally-stored output parameter visitor.
+  OutputParameterVisitor outputParameterVisitor;
+
+  //! Locally-stored delete visitor.
+  DeleteVisitor deleteVisitor;
+
+  //! Locally-stored output width visitor.
+  OutputWidthVisitor outputWidthVisitor;
+
+  //! Locally-stored output height visitor.
+  OutputHeightVisitor outputHeightVisitor;
+}; // class Highway
+
+} // namespace ann
+} // namespace mlpack
+
+// Include implementation.
+#include "highway_impl.hpp"
+
+#endif

src/mlpack/methods/ann/layer/highway_impl.hpp

@@ -0,0 +1,238 @@
+/**
+ * @file methods/ann/layer/highway_impl.hpp
+ * @author Konstantin Sidorov
+ * @author Saksham Bansal
+ *
+ * Implementation of Highway layer.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_HIGHWAY_IMPL_HPP
+#define MLPACK_METHODS_ANN_LAYER_HIGHWAY_IMPL_HPP
+
+// In case it hasn't yet been included.
+#include "highway.hpp"
+
+#include "../visitor/forward_visitor.hpp"
+#include "../visitor/backward_visitor.hpp"
+#include "../visitor/gradient_visitor.hpp"
+#include "../visitor/set_input_height_visitor.hpp"
+#include "../visitor/set_input_width_visitor.hpp"
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+Highway<InputDataType, OutputDataType, CustomLayers...>::Highway() :
+    inSize(0),
+    model(true),
+    reset(false),
+    width(0),
+    height(0)
+{
+  // Nothing to do here.
+}
+
+template<
+    typename InputDataType, typename OutputDataType, typename... CustomLayers>
+Highway<InputDataType, OutputDataType, CustomLayers...>::Highway(
+    const size_t inSize,
+    const bool model) :
+    inSize(inSize),
+    model(model),
+    reset(false),
+    width(0),
+    height(0)
+{
+  weights.set_size(inSize * inSize + inSize, 1);
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+Highway<InputDataType, OutputDataType, CustomLayers...>::~Highway()
+{
+  if (!model)
+  {
+    for (size_t i = 0; i < network.size(); ++i)
+    {
+      if (networkOwnerships[i])
+        boost::apply_visitor(deleteVisitor, network[i]);
+    }
+  }
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+void Highway<InputDataType, OutputDataType, CustomLayers...>::Reset()
+{
+  transformWeight = arma::mat(weights.memptr(), inSize, inSize, false, false);
+  transformBias = arma::mat(weights.memptr() + transformWeight.n_elem,
+      inSize, 1, false, false);
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+template<typename eT>
+void Highway<InputDataType, OutputDataType, CustomLayers...>::Forward(
+    const arma::Mat<eT>& input, arma::Mat<eT>& output)
+{
+  boost::apply_visitor(ForwardVisitor(input,
+      boost::apply_visitor(outputParameterVisitor, network.front())),
+      network.front());
+
+  if (!reset)
+  {
+    if (boost::apply_visitor(outputWidthVisitor, network.front()) != 0)
+    {
+      width = boost::apply_visitor(outputWidthVisitor, network.front());
+    }
+
+    if (boost::apply_visitor(outputHeightVisitor, network.front()) != 0)
+    {
+      height = boost::apply_visitor(outputHeightVisitor, network.front());
+    }
+  }
+
+  for (size_t i = 1; i < network.size(); ++i)
+  {
+    if (!reset)
+    {
+      // Set the input width.
+      boost::apply_visitor(SetInputWidthVisitor(width), network[i]);
+
+      // Set the input height.
+      boost::apply_visitor(SetInputHeightVisitor(height), network[i]);
+    }
+
+    boost::apply_visitor(ForwardVisitor(boost::apply_visitor(
+        outputParameterVisitor, network[i - 1]),
+        boost::apply_visitor(outputParameterVisitor, network[i])),
+        network[i]);
+
+    if (!reset)
+    {
+      // Get the output width.
+      if (boost::apply_visitor(outputWidthVisitor, network[i]) != 0)
+      {
+        width = boost::apply_visitor(outputWidthVisitor, network[i]);
+      }
+
+      // Get the output height.
+      if (boost::apply_visitor(outputHeightVisitor, network[i]) != 0)
+      {
+        height = boost::apply_visitor(outputHeightVisitor, network[i]);
+      }
+    }
+  }
+  if (!reset)
+  {
+    reset = true;
+  }
+
+  output = boost::apply_visitor(outputParameterVisitor, network.back());
+
+  if (arma::size(output) != arma::size(input))
+  {
+    Log::Fatal << "The sizes of the output and input matrices of the Highway"
+        << " network should be equal. Please examine the network layers.";
+  }
+
+  transformGate = transformWeight * input;
+  transformGate.each_col() += transformBias;
+  transformGateActivation = 1.0 /(1 + arma::exp(-transformGate));
+  inputParameter = input;
+  networkOutput = output;
+  output = (output % transformGateActivation) +
+      (input % (1 - transformGateActivation));
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+template<typename eT>
+void Highway<InputDataType, OutputDataType, CustomLayers...>::Backward(
+    const arma::Mat<eT>& /* input */,
+    const arma::Mat<eT>& gy,
+    arma::Mat<eT>& g)
+{
+  arma::Mat<eT> gyTransform = gy % transformGateActivation;
+  boost::apply_visitor(BackwardVisitor(boost::apply_visitor(
+      outputParameterVisitor, network.back()),
+      gyTransform,
+      boost::apply_visitor(deltaVisitor, network.back())),
+      network.back());
+
+  for (size_t i = 2; i < network.size() + 1; ++i)
+  {
+    boost::apply_visitor(BackwardVisitor(boost::apply_visitor(
+        outputParameterVisitor, network[network.size() - i]),
+        boost::apply_visitor(deltaVisitor, network[network.size() - i + 1]),
+        boost::apply_visitor(deltaVisitor,
+        network[network.size() - i])), network[network.size() - i]);
+  }
+
+  g = boost::apply_visitor(deltaVisitor, network.front());
+
+  transformGateError = gy % (networkOutput - inputParameter) %
+      transformGateActivation % (1.0 - transformGateActivation);
+  g += transformWeight.t() * transformGateError;
+  g += gy % (1 - transformGateActivation);
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+template<typename eT>
+void Highway<InputDataType, OutputDataType, CustomLayers...>::Gradient(
+    const arma::Mat<eT>& input,
+    const arma::Mat<eT>& error,
+    arma::Mat<eT>& gradient)
+{
+  arma::Mat<eT> errorTransform = error % transformGateActivation;
+  boost::apply_visitor(GradientVisitor(boost::apply_visitor(
+      outputParameterVisitor, network[network.size() - 2]),
+      errorTransform), network.back());
+
+  for (size_t i = 2; i < network.size(); ++i)
+  {
+    boost::apply_visitor(GradientVisitor(boost::apply_visitor(
+        outputParameterVisitor, network[network.size() - i - 1]),
+        boost::apply_visitor(deltaVisitor, network[network.size() - i + 1])),
+        network[network.size() - i]);
+  }
+
+  boost::apply_visitor(GradientVisitor(input,
+      boost::apply_visitor(deltaVisitor, network[1])), network.front());
+
+  gradient.submat(0, 0, transformWeight.n_elem - 1, 0) = arma::vectorise(
+      transformGateError * input.t());
+  gradient.submat(transformWeight.n_elem, 0, gradient.n_elem - 1, 0) =
+      arma::sum(transformGateError, 1);
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+template<typename Archive>
+void Highway<InputDataType, OutputDataType, CustomLayers...>::serialize(
+    Archive& ar, const uint32_t /* version */)
+{
+  // If loading, delete the old layers and set size for weights.
+  if (cereal::is_loading<Archive>())
+  {
+    for (LayerTypes<CustomLayers...>& layer : network)
+    {
+      boost::apply_visitor(deleteVisitor, layer);
+    }
+    weights.set_size(inSize * inSize + inSize, 1);
+  }
+
+  ar(CEREAL_NVP(model));
+  ar(CEREAL_VECTOR_VARIANT_POINTER(network));
+}
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/layer/join.hpp

@@ -17,30 +17,24 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-// TODO: should we clarify the comments?  This seems to join together points of
-// a different batch
-// TODO: I don't understand this layer well enough to update it...
 /**
  * Implementation of the Join module class. The Join class accumulates
  * the output of various modules.
  *
- * @tparam InputType Type of the input data (arma::colvec, arma::mat,
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
- * @tparam OutputType Type of the output data (arma::colvec, arma::mat,
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
  */
 template<
-    typename InputType = arma::mat,
-    typename OutputType = arma::mat
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
 >
-class JoinType : public Layer<InputType, OutputType>
+class Join
 {
  public:
-  //! Create the JoinType object.
-  JoinType();
-
-  //! Clone the JoinType object. This handles polymorphism correctly.
-  JoinType* Clone() const { return new JoinType(*this); }
+  //! Create the Join object.
+  Join();
 
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function
@@ -49,6 +43,7 @@ class JoinType : public Layer<InputType, OutputType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
+  template<typename InputType, typename OutputType>
   void Forward(const InputType& input, OutputType& output);
 
   /**
@@ -60,19 +55,20 @@ class JoinType : public Layer<InputType, OutputType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const InputType& /* input */,
-                const OutputType& gy,
-                OutputType& g);
+  template<typename eT>
+  void Backward(const arma::Mat<eT>& /* input */,
+                const arma::Mat<eT>& gy,
+                arma::Mat<eT>& g);
 
-  // This layer simply flattens its input into a vector.
-  const std::vector<size_t> OutputDimensions() const
-  {
-    // TODO: it's not clear what to do here
-    std::vector<size_t> result(inputDimensions.size(), 0);
-    result[0] = std::accumulate(inputDimensions.begin(), inputDimensions.end(),
-        0);
-    return result;
-  }
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
 
   /**
    * Serialize the layer.
@@ -86,10 +82,13 @@ class JoinType : public Layer<InputType, OutputType>
 
   //! Locally-stored number of input cols.
   size_t inSizeCols;
-}; // class JoinType
 
-//Standard Join layer.
-typedef JoinType<arma::mat, arma::mat> Join;
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+}; // class Join
 
 } // namespace ann
 } // namespace mlpack