Update and release version 2.10.0.

use fast solve

.github/ISSUE_TEMPLATE/0-bug.md

@@ -0,0 +1,40 @@
+---
+name: Bug report
+about: Use this template for reporting a bug that you have found in ensmallen.
+title: ''
+labels: 't: bug report, s: unanswered'
+assignees: ''
+
+---
+
+<!--
+
+Welcome!  Please fill out the template below; that makes it easier for us to 
+quickly figure out what the issue is and solve it.  Thanks!
+
+-->
+
+#### Issue description
+
+<!-- Describe your issue here. -->
+
+#### Your environment
+
+ * version of ensmallen:
+ * operating system:
+ * compiler:
+ * version of Armadillo:
+ * any other environment information you think is relevant:
+
+#### Steps to reproduce
+
+<!-- Tell us how to reproduce the issue; please provide a working example if
+possible! -->
+
+#### Expected behavior
+
+<!-- Tell us what should happen. -->
+
+#### Actual behavior
+
+<!-- Tell us what happened instead. -->

.github/ISSUE_TEMPLATE/1-documentation.md

@@ -0,0 +1,26 @@
+---
+name: Documentation issue
+about: Use this template to report an issue you've found with the documentation.
+title: ''
+labels: 't: bug report, c: documentation, s: unanswered'
+assignees: ''
+
+---
+
+<!--
+
+Welcome!  Unfortunately not all documentation is perfect, and if you're opening
+a documentation issue we are interested in fixing it.  Please fill out the
+template below so that we can solve the problem more quickly; or, alternately,
+open a PR with a fix, if you like.
+
+-->
+
+#### Problem location
+
+<!-- Link to incorrect website or location of source file with bad
+documentation. -->
+
+#### Description of problem
+
+<!-- Tell us what is wrong with the documentation so we can fix it. -->

.github/ISSUE_TEMPLATE/2-question.md

@@ -0,0 +1,18 @@
+---
+name: Question
+about: Use this template for other problems, requests, or questions.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+<!--
+
+Welcome!  If you have a question you'd like to ask, you can do it here or on the
+mlpack mailing list; see also http://mlpack.org/help.html.
+
+If you're looking for how to get involved and contribute, there's no need to
+open an issue---you can see http://www.mlpack.org/involved.html instead.
+
+-->

.gitignore

@@ -0,0 +1 @@
+build/

.travis.yml

@@ -8,17 +8,17 @@ env:
 
 before_install:
   - sudo apt-get update
-  - sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++
+  - sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ xz-utils
   - if [ $ARMADILLO == "latest" ]; then
       curl https://ftp.fau.de/macports/distfiles/armadillo/`curl https://ftp.fau.de/macports/distfiles/armadillo/ -- | grep '.tar.xz' | sed 's/^.*<a href="\(armadillo-[0-9]*.[0-9]*.[0-9]*.tar.xz\)".*$/\1/' | tail -1` | tar xvJ && cd armadillo*;
     fi
   - if [ $ARMADILLO == "minimum" ]; then
-      curl https://ftp.fau.de/macports/distfiles/armadillo/armadillo-6.500.5.tar.gz | tar xvz && cd armadillo*;
+      curl https://ftp.fau.de/macports/distfiles/armadillo/armadillo-8.400.0.tar.xz | tar -xvJ && cd armadillo*;
     fi
   - cmake . && make && sudo make install && cd ..
 
 install:
-  - mkdir build && cd build && cmake .. && make -j2
+  - mkdir build && cd build && cmake -DCMAKE_CXX_FLAGS="-Werror" -DCMAKE_C_FLAGS="-Werror" .. && make -j2
 
 script:
   - CTEST_OUTPUT_ON_FAILURE=1 travis_wait 30 ctest -j2

CMakeLists.txt

@@ -41,11 +41,18 @@ else ()
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-pragmas")
 endif ()
 
+# Set the CFLAGS and CXXFLAGS depending on the options the user specified.
+if(CMAKE_COMPILER_IS_GNUCC OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wpedantic -Wunused-parameter")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wpedantic -Wunused-parameter")
+endif()
+
 # The only dependency we need is Armadillo.
 #
 # We keep the minimum version in sync with mlpack, otherwise we could have
 # irritating compatibility issues.
-find_package(Armadillo 6.500.0 REQUIRED)
+find_package(Armadillo 8.400.0 REQUIRED)
 include_directories(BEFORE "${ARMADILLO_INCLUDE_DIR}")
 include_directories(BEFORE "${CMAKE_SOURCE_DIR}/include/")
 
@@ -59,4 +66,4 @@ install(FILES ${CMAKE_SOURCE_DIR}/include/ensmallen.hpp
 
 enable_testing()
 
-add_subdirectory(tests)
+add_subdirectory(tests)

COPYRIGHT.txt

@@ -35,6 +35,7 @@ Copyright:
   Copyright 2018, Shikhar Jaiswal <jaiswalshikhar87@gmail.com>
   Copyright 2018, Conrad Sanderson
   Copyright 2018, Dan Timson
+  Copyright 2019, Rahul Ganesh Prabhu
   
 License: BSD-3-clause
   All rights reserved.

HISTORY.md

@@ -1,59 +1,153 @@
-### ensmallen 1.13.2
+### ensmallen 2.10.0: "Fried Chicken"
+###### 2019-09-07
+  * All `Optimize()` functions now take any matrix type; so, e.g., `arma::fmat`
+    or `arma::sp_mat` can be used for optimization.  See the documentation for
+    more details ([#113](https://github.com/mlpack/ensmallen/pull/113),
+    [#119](https://github.com/mlpack/ensmallen/pull/119)).
+
+  * Introduce callback support.  Callbacks can be appended as the last arguments
+    of an `Optimize()` call, and can perform custom behavior at different points
+    during the optimization.  See the documentation for more details
+    ([#119](https://github.com/mlpack/ensmallen/pull/119)).
+
+  * Slight speedups for `FrankWolfe` optimizer
+    ([#127](https://github.com/mlpack/ensmallen/pull/127)).
+
+### ensmallen 1.16.2: "Loud Alarm Clock"
+###### 2019-08-12
+  * Fix PSO return type bug
+    ([#126](https://github.com/mlpack/ensmallen/pull/126)).
+
+### ensmallen 1.16.1: "Loud Alarm Clock"
+###### 2019-08-11
+  * Update HISTORY.md to use Markdown links to the PR and add release names.
+
+  * Fix PSO return type bug
+    ([#124](https://github.com/mlpack/ensmallen/pull/124)).
+
+### ensmallen 1.16.0: "Loud Alarm Clock"
+###### 2019-08-09
+  * Add option to avoid computing exact objective at the end of the optimization
+    ([#109](https://github.com/mlpack/ensmallen/pull/109)).
+
+  * Fix handling of curvature for BigBatchSGD
+    ([#118](https://github.com/mlpack/ensmallen/pull/118)).
+
+  * Reduce runtime of tests
+    ([#118](https://github.com/mlpack/ensmallen/pull/118)).
+
+  * Introduce local-best particle swarm optimization, `LBestPSO`, for
+    unconstrained optimization problems
+    ([#86](https://github.com/mlpack/ensmallen/pull/86)).
+
+### ensmallen 1.15.1: "Wrong Side Of The Road"
+###### 2019-05-22
+  * Fix `-Wreorder` in `qhadam` warning
+    ([#115](https://github.com/mlpack/ensmallen/pull/115)).
+
+  * Fix `-Wunused-private-field` warning in `spsa`
+    ([#115](https://github.com/mlpack/ensmallen/pull/115)).
+
+  * Add more warning output for gcc/clang
+    ([#116](https://github.com/mlpack/ensmallen/pull/116)).
+
+### ensmallen 1.15.0: "Wrong Side Of The Road"
+###### 2019-05-14
+  * Added QHAdam and QHSGD optimizers
+    ([#81](https://github.com/mlpack/ensmallen/pull/81)).
+
+### ensmallen 1.14.4: "Difficult Crimp"
+###### 2019-05-12
+  * Fixes for BigBatchSGD ([#91](https://github.com/mlpack/ensmallen/pull/91)).
+
+### ensmallen 1.14.3: "Difficult Crimp"
+###### 2019-05-06
+  * Handle `eig_sym()` failures correctly
+    ([#100](https://github.com/mlpack/ensmallen/pull/100)).
+
+### ensmallen 1.14.2: "Difficult Crimp"
+###### 2019-03-14
+  * SPSA test tolerance fix
+    ([#97](https://github.com/mlpack/ensmallen/pull/97)).
+
+  * Minor documentation fixes (#95, #98).
+
+  * Fix newlines at end of file
+    ([#92](https://github.com/mlpack/ensmallen/pull/92)).
+
+### ensmallen 1.14.1: "Difficult Crimp"
+###### 2019-03-09
+  * Fixes for SPSA ([#87](https://github.com/mlpack/ensmallen/pull/87)).
+
+  * Optimized CNE and DE ([#90](https://github.com/mlpack/ensmallen/pull/90)).
+    Changed initial population generation in CNE to be a normal distribution
+    about the given starting point, which should accelerate convergence.
+
+### ensmallen 1.14.0: "Difficult Crimp"
+###### 2019-02-20
+  * Add DE optimizer ([#77](https://github.com/mlpack/ensmallen/pull/77)).
+
+  * Fix for Cholesky decomposition in CMAES
+    ([#83](https://github.com/mlpack/ensmallen/pull/83)).
+
+### ensmallen 1.13.2: "Coronavirus Invasion"
 ###### 2019-02-18
- * Minor documentation fixes (#82).
+ * Minor documentation fixes ([#82](https://github.com/mlpack/ensmallen/pull/82)).
 
-### ensmallen 1.13.1
+### ensmallen 1.13.1: "Coronavirus Invasion"
 ###### 2019-01-24
- * Fix -Wreorder warning (#75).
+ * Fix -Wreorder warning ([#75](https://github.com/mlpack/ensmallen/pull/75)).
 
-### ensmallen 1.13.0
+### ensmallen 1.13.0: "Coronavirus Invasion"
 ###### 2019-01-14
- * Enhance options for AugLagrangian optimizer (#66).
+ * Enhance options for AugLagrangian optimizer
+   ([#66](https://github.com/mlpack/ensmallen/pull/66)).
 
- * Add SPSA optimizer (#69).
+ * Add SPSA optimizer ([#69](https://github.com/mlpack/ensmallen/pull/69)).
 
-### ensmallen 1.12.2
+### ensmallen 1.12.2: "New Year's Party"
 ###### 2019-01-05
  * Fix list of contributors.
 
-### ensmallen 1.12.1
+### ensmallen 1.12.1: "New Year's Party"
 ###### 2019-01-03
  * Make sure all files end with newlines.
 
-### ensmallen 1.12.0
+### ensmallen 1.12.0: "New Year's Party"
 ###### 2018-12-30
  * Add link to ensmallen PDF to README.md.
 
  * Minor documentation fixes.  Remove too-verbose documentation from source for
-   each optimizer (#61).
+   each optimizer ([#61](https://github.com/mlpack/ensmallen/pull/61)).
 
- * Add FTML optimizer (#48).
+ * Add FTML optimizer ([#48](https://github.com/mlpack/ensmallen/pull/48)).
 
- * Add SWATS optimizer (#42).
+ * Add SWATS optimizer ([#42](https://github.com/mlpack/ensmallen/pull/42)).
 
- * Add Padam optimizer (#46).
+ * Add Padam optimizer ([#46](https://github.com/mlpack/ensmallen/pull/46)).
 
- * Add Eve optimizer (#45).
+ * Add Eve optimizer ([#45](https://github.com/mlpack/ensmallen/pull/45)).
 
- * Add ResetPolicy() to SGD-like optimizers (#60).
+ * Add ResetPolicy() to SGD-like optimizers
+   ([#60](https://github.com/mlpack/ensmallen/pull/60)).
 
-### ensmallen 1.11.1
+### ensmallen 1.11.1: "Jet Lag"
 ###### 2018-11-29
  * Minor documentation fixes.
 
-### ensmallen 1.11.0
+### ensmallen 1.11.0: "Jet Lag"
 ###### 2018-11-28
  * Add WNGrad optimizer.
 
  * Fix header name in documentation samples.
 
-### ensmallen 1.10.1
+### ensmallen 1.10.1: "Corporate Catabolism"
 ###### 2018-11-16
  * Fixes for GridSearch optimizer.
 
  * Include documentation with release.
 
-### ensmallen 1.10.0
+### ensmallen 1.10.0: "Corporate Catabolism"
 ###### 2018-10-20
  * Initial release.
 

doc/callbacks.md

@@ -0,0 +1,484 @@
+Callbacks in mlpack are methods that are called at various states during the
+optimization process, which can be used to implement and control behaviors such
+as:
+
+* Changing the learning rate.
+* Printing of the current objective.
+* Sending a message when the optimization hits a specific state such us a minimal objective.
+
+Callbacks can be passed as an argument to the `Optimize()` function:
+
+```c++
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+
+MomentumSGD optimizer(0.01, 32, 100000, 1e-5, true, MomentumUpdate(0.5));
+
+// Pass the built-in *PrintLoss* callback as the last argument to the
+// *Optimize()* function.
+optimizer.Optimize(f, coordinates, PrintLoss());
+```
+
+Passing multiple callbacks is just the same as passing a single callback:
+
+```c++
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+
+MomentumSGD optimizer(0.01, 32, 100000, 1e-5, true, MomentumUpdate(0.5));
+
+// Pass the built-in *PrintLoss* and *EarlyStopAtMinLoss* callback as the last
+// argument to the *Optimize()* function.
+optimizer.Optimize(f, coordinates, PrintLoss(), EarlyStopAtMinLoss());
+```
+
+It is also possible to pass a callback instantiation that allows accessing of
+internal callback parameters at a later state:
+
+```c++
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+
+MomentumSGD optimizer(0.01, 32, 100000, 1e-5, true, MomentumUpdate(0.5));
+
+// Create an instantiation of the built-in *StoreBestCoordinates* callback,
+// which will store the best objective and the corresponding model parameter
+// that can be accessed later.
+StoreBestCoordinates<> callback;
+
+// Pass an instantiation of the built-in *StoreBestCoordinates* callback as the
+// last argument to the *Optimize()* function.
+optimizer.Optimize(f, coordinates, callback);
+
+// Print the minimum objective that is stored inside the *StoreBestCoordinates*
+// callback that was passed to the *Optimize()* call.
+std::cout << callback.BestObjective() << std::endl;
+```
+
+## Built-in Callbacks
+
+### EarlyStopAtMinLoss
+
+Stops the optimization process if the loss stops decreasing or no improvement
+has been made.
+
+#### Constructors
+
+ * `EarlyStopAtMinLoss()`
+ * `EarlyStopAtMinLoss(`_`patience`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `size_t` | **`patience`** | The number of epochs to wait after the minimum loss has been reached. | `10` |
+
+#### Examples:
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+optimizer.Optimize(f, coordinates, EarlyStopAtMinLoss());
+```
+
+### PrintLoss
+
+Callback that prints loss to stdout or a specified output stream.
+
+#### Constructors
+
+ * `PrintLoss()`
+ * `PrintLoss(`_`output`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `std::ostream` | **`output`** | Ostream which receives output from this object. | `stdout` |
+
+#### Examples:
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+optimizer.Optimize(f, coordinates, PrintLoss());
+```
+
+### ProgressBar
+
+Callback that prints a progress bar to stdout or a specified output stream.
+
+#### Constructors
+
+ * `ProgressBar()`
+ * `ProgressBar(`_`width`_`)`
+ * `ProgressBar(`_`width, output`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `size_t` | **`width`** | Width of the bar. | `70` |
+| `std::ostream` | **`output`** | Ostream which receives output from this object. | `stdout` |
+
+#### Examples:
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+optimizer.Optimize(f, coordinates, ProgressBar());
+```
+
+### StoreBestCoordinates
+
+Callback that stores the model parameter after every epoch if the objective
+decreased.
+
+#### Constructors
+
+ * `StoreBestCoordinates<`_`ModelMatType`_`>()`
+
+The _`ModelMatType`_ template parameter refers to the matrix type of the model
+parameter.
+
+#### Attributes
+
+The stored model parameter can be accessed via the member method
+`BestCoordinates()` and the best objective via `BestObjective()`.
+
+#### Examples:
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+
+StoreBestCoordinates<arma::mat> cb;
+optimizer.Optimize(f, coordinates, cb);
+
+std::cout << "The optimized model found by AdaDelta has the "
+      << "parameters " << cb.BestCoordinatest();
+```
+
+## Callback States
+
+Callbacks are called at different states during the optimization process:
+
+* At the beginning and end of the optimization process.
+* After any call to `Evaluate()` and `EvaluateConstraint`.
+* After any call to `Gradient()` and `GradientConstraint`.
+* At the start and end of an epoch.
+
+Each callback provides optimization relevant information that can be accessed or
+modified.
+
+### BeginOptimization
+
+Called at the beginning of the optimization process.
+
+ * `BeginOptimization(`_`optimizer, function, coordinates`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+
+### EndOptimization
+
+Called at the end of the optimization process.
+
+ * `EndOptimization(`_`optimizer, function, coordinates`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+
+### Evaluate
+
+Called after any call to `Evaluate()`.
+
+ * `Evaluate(`_`optimizer, function, coordinates, objective`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+| `double` | **`objective`** | Objective value of the current point. |
+
+### EvaluateConstraint
+
+ Called after any call to `EvaluateConstraint()`.
+
+ * `EvaluateConstraint(`_`optimizer, function, coordinates, constraint, constraintValue`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+| `size_t` | **`constraint`** | The index of the constraint. |
+| `double` | **`constraintValue`** | Constraint value of the current point. |
+
+### Gradient
+
+ Called after any call to `Gradient()`.
+
+ * `Gradient(`_`optimizer, function, coordinates, gradient`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+| `GradType` | **`gradient`** | Matrix that holds the gradient. |
+
+### GradientConstraint
+
+ Called after any call to `GradientConstraint()`.
+
+ * `GradientConstraint(`_`optimizer, function, coordinates, constraint, gradient`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+| `size_t` | **`constraint`** | The index of the constraint. |
+| `GradType` | **`gradient`** | Matrix that holds the gradient. |
+
+### BeginEpoch
+
+Called at the beginning of a pass over the data. The objective may be exact or
+an estimate depending on `exactObjective` value.
+
+ * `BeginEpoch(`_`optimizer, function, coordinates, epoch, objective`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+| `size_t` | **`epoch`** | The index of the current epoch. |
+| `double` | **`objective`** | Objective value of the current point. |
+
+### EndEpoch
+
+Called at the end of a pass over the data. The objective may be exact or
+an estimate depending on `exactObjective` value.
+
+ * `EndEpoch(`_`optimizer, function, coordinates, epoch, objective`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+| `size_t` | **`epoch`** | The index of the current epoch. |
+| `double` | **`objective`** | Objective value of the current point. |
+
+## Custom Callbacks
+
+### Learning rate scheduling
+
+Setting the learning rate is crucially important when training because it
+controls both the speed of convergence and the ultimate performance of the
+model. One of the simplest learning rate strategies is to have a fixed learning
+rate throughout the training process. Choosing a small learning rate allows the
+optimizer to find good solutions, but this comes at the expense of limiting the
+initial speed of convergence. To overcome this tradeoff, changing the learning
+rate as more epochs have passed is commonly done in model training. The
+`Evaluate` method in combination with the ``StepSize`` method of the optimizer
+can be used to update the variables.
+
+Example code showing how to implement a custom callback to change the learning
+rate is given below.
+
+```c++
+class ExponentialDecay
+{
+  // Set up the exponential decay learning rate scheduler with the user
+  // specified decay value.
+  ExponentialDecay(const double decay) : decay(decay), learningRate(0) { }
+
+
+  // Callback function called at the start of the optimization process.
+  // In this example we will use this to save the initial learning rate.
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void BeginOptimization(OptimizerType& /* optimizer */,
+                         FunctionType& /* function */,
+                         MatType& /* coordinates */)
+  {
+    // Save the initial learning rate.
+    learningRate = optimizer.StepSize();
+  }
+
+  // Callback function called at the end of a pass over the data. We are only
+  // interested in the current epoch and the optimizer, we ignore the rest.
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void EndEpoch(OptimizerType& optimizer,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                const size_t epoch,
+                const double objective)
+  {
+    // Update the learning rate.
+    optimizer.StepSize() = learningRate * (1.0 - std::pow(decay,
+        (double) epoch));
+  }
+
+  double learningRate;
+};
+
+int main()
+{
+  // First, generate some random data, with 10000 points and 10 dimensions.
+  // This data has no pattern and as such will make a model that's not very
+  // useful---but the purpose here is just demonstration. :)
+  //
+  // For a more "real world" situation, load a dataset from file using X.load()
+  // and y.load() (but make sure the matrix is column-major, so that each
+  // observation/data point corresponds to a *column*, *not* a row.
+  arma::mat data(10, 10000, arma::fill::randn);
+  arma::rowvec responses(10000, arma::fill::randn);
+
+  // Create a starting point for our optimization randomly.  The model has 10
+  // parameters, so the shape is 10x1.
+  arma::mat startingPoint(10, 1, arma::fill::randn);
+
+  // Construct the objective function.
+  LinearRegressionFunction lrf(data, responses);
+  arma::mat lrfParams(startingPoint);
+
+  // Create the StandardSGD optimizer with specified parameters.
+  // The ens::StandardSGD type can be replaced with any ensmallen optimizer
+  //that can handle differentiable functions.
+  StandardSGD optimizer(0.001, 1, 0, 1e-15, true);
+
+  // Use the StandardSGD optimizer with specified parameters to minimize the
+  // LinearRegressionFunction and pass the *exponential decay*
+  // callback function from above.
+  optimizer.Optimize(lrf, lrfParams, ExponentialDecay(0.01));
+
+  // Print the trained model parameter.
+  std::cout << lrfParams.t();
+}
+```
+
+### Early stopping at minimum loss
+
+Early stopping is a technique for controlling overfitting in machine learning
+models, especially neural networks, by stopping the optimization process before
+the model has trained for the maximum number of iterations.
+
+Example code showing how to implement a custom callback to stop the optimization
+when the minimum of loss has been reached is given below.
+
+```c++
+#include <ensmallen.hpp>
+
+// This class implements early stopping at minimum loss callback function to
+// terminate the optimization process early if the loss stops decreasing.
+class EarlyStop
+{
+ public:
+  // Set up the early stop at min loss class, which keeps track of the minimum
+  // loss.
+  EarlyStop() : bestObjective(std::numeric_limits<double>::max()) { }
+
+  // Callback function called at the end of a pass over the data, which provides
+  // the current objective. We are only interested in the objective and ignore
+  // the rest.
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void EndEpoch(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                const size_t /* epoch */,
+                const double objective)
+  {
+    // Check if the given objective is lower as the previous objective.
+    if (objective < bestObjective)
+    {
+      // Update the local objective.
+      bestObjective = objective;
+    }
+    else
+    {
+      // Stop the optimization process.
+      return true;
+    }
+
+    // Do not stop the optimization process.
+    return false;
+  }
+
+  // Locally-stored best objective.
+  double bestObjective;
+};
+
+int main()
+{
+  // First, generate some random data, with 10000 points and 10 dimensions.
+  // This data has no pattern and as such will make a model that's not very
+  // useful---but the purpose here is just demonstration. :)
+  //
+  // For a more "real world" situation, load a dataset from file using X.load()
+  // and y.load() (but make sure the matrix is column-major, so that each
+  // observation/data point corresponds to a *column*, *not* a row.
+  arma::mat data(10, 10000, arma::fill::randn);
+  arma::rowvec responses(10000, arma::fill::randn);
+
+  // Create a starting point for our optimization randomly.  The model has 10
+  // parameters, so the shape is 10x1.
+  arma::mat startingPoint(10, 1, arma::fill::randn);
+
+  // Construct the objective function.
+  LinearRegressionFunction lrf(data, responses);
+  arma::mat lrfParams(startingPoint);
+
+  // Create the L_BFGS optimizer with default parameters.
+  // The ens::L_BFGS type can be replaced with any ensmallen optimizer that can
+  // handle differentiable functions.
+  ens::L_BFGS lbfgs;
+
+  // Use the L_BFGS optimizer with default parameters to minimize the
+  // LinearRegressionFunction and pass the *early stopping at minimum loss*
+  // callback function from above.
+  lbfgs.Optimize(lrf, lrfParams, EarlyStop());
+
+  // Print the trained model parameter.
+  std::cout << lrfParams.t();
+}
+```
+
+Note that we have simply passed an instantiation of `EarlyStop` the
+rest is handled inside the optimizer.
+
+ensmallen provides a more complete and general implementation of a
+[early stopping](#EarlyStopAtMinLoss) at minimum loss callback function.

doc/function_types.md

@@ -23,6 +23,8 @@ The following optimizers can be used to optimize an arbitrary function:
 
  - [Simulated Annealing](#simulated-annealing-sa)
  - [CNE](#cne)
+ - [DE](#de)
+ - [SPSA](#simultaneous-perturbation-stochastic-approximation-spsa)
 
 Each of these optimizers has an `Optimize()` function that is called as
 `Optimize(f, x)` where `f` is the function to be optimized (which implements
@@ -30,7 +32,7 @@ Each of these optimizers has an `Optimize()` function that is called as
 `Optimize()` is called, `x` will hold the final result of the optimization
 (that is, the best `x` found that minimizes `f(x)`).
 
-#### Example: Linear Regression
+#### Example: squared function optimization
 
 An example program that implements the objective function f(x) = 2 |x|^2 is
 shown below, using the simulated annealing optimizer.
@@ -50,7 +52,7 @@ class SquaredFunction
 
 int main()
 {
-  // The minimum is at x = [0 0 0].  Our initial point is chosen to be 
+  // The minimum is at x = [0 0 0].  Our initial point is chosen to be
   // [1.0, -1.0, 1.0].
   arma::mat x("1.0 -1.0 1.0");
 
@@ -144,7 +146,7 @@ class LinearRegressionFunction
  public:
   // Construct the object with the given data matrix and responses.
   LinearRegressionFunction(const arma::mat& dataIn,
-                           const arma::rowvec& responsesIn) : 
+                           const arma::rowvec& responsesIn) :
       data(dataIn), responses(responsesIn) { }
 
   // Return the objective function for model parameters x.
@@ -171,13 +173,13 @@ class LinearRegressionEWGFunction
 {
  public:
   // Construct the object with the given data matrix and responses.
-  LinearRegressionEWGFunction(const arma::mat& dataIn, 
-                              const arma::rowvec& responsesIn) : 
+  LinearRegressionEWGFunction(const arma::mat& dataIn,
+                              const arma::rowvec& responsesIn) :
       data(dataIn), responses(responsesIn) { }
 
   // Simultaneously compute both the objective function and gradient for model
   // parameters x.  Note that this is faster than implementing Evaluate() and
-  // Gradient() individually because it caches the computation of 
+  // Gradient() individually because it caches the computation of
   // (responses - x.t() * data)!
   double EvaluateWithGradient(const arma::mat& x, arma::mat& g)
   {
@@ -286,7 +288,7 @@ f(x) = f_0(x) + f_1(x) + ... + f_N(x).
 ```
 
 In this function type, we assume the gradient `f'(x)` is not computable.  If it
-is, see [#Differentiable-separable-functions].
+is, see [differentiable separable functions](#differentiable-separable-functions).
 
 For machine learning tasks, the objective function may be, e.g., the sum of a
 function taken across many data points.  Implementing an arbitrary separable
@@ -333,7 +335,7 @@ intensive operation for your objective function, it may be best to avoid using
 a non-separable arbitrary function optimizer.
 
 **Note**: if possible, it's often better to try and use a gradient-based
-approach.  See [#Differentiable-separable-functions]
+approach.  See [differentiable separable functions](#differentiable-separable-functions)
 for separable f(x) where the gradient f'(x) can be computed.
 
 The example program below demonstrates the implementation and use of an
@@ -481,7 +483,7 @@ class ArbitrarySeparableFunctionType
   //
   // Given parameters x and a matrix g, return the sum of the individual
   // functions f_i(x) + ... + f_{i + batchSize - 1}(x), and store the sum of
-  // the gradient of individual functions f'_i(x) + ... + 
+  // the gradient of individual functions f'_i(x) + ... +
   // f'_{i + batchSize - 1}(x) into the provided matrix g.  g should have the
   // same size (rows, columns) as x.  i will always be greater than 0, and i +
   // batchSize will be less than or equal to the value of NumFunctions().
@@ -516,6 +518,8 @@ The following optimizers can be used with differentiable functions:
  - [NadaMax](#nadamax)
  - [NesterovMomentumSGD](#nesterov-momentum-sgd)
  - [OptimisticAdam](#optimisticadam)
+ - [QHAdam](#qhadam)
+ - [QHSGD](#qhsgd)
  - [RMSProp](#rmsprop)
  - [SARAH/SARAH+](#stochastic-recursive-gradient-algorithm-sarahsarah)
  - [SGD](#standard-sgd)
@@ -846,7 +850,7 @@ Example code showing how to solve an SDP is given below.
 ```c++
 int main()
 {
-  // We will build a toy semidefinite program and then use the PrimalDualSolver to find a solution 
+  // We will build a toy semidefinite program and then use the PrimalDualSolver to find a solution
 
   // The semi-definite constraint looks like:
   //
@@ -946,3 +950,93 @@ int main()
   std::cout << "SDP optimized with objective " << obj << "." << std::endl;
 }
 ```
+
+## Alternate matrix types
+
+All of the examples above (and throughout the rest of the documentation)
+generally assume that the matrix being optimized has type `arma::mat`.  But
+ensmallen's optimizers are capable of optimizing more types than just dense
+Armadillo matrices.  In fact, the full signature of each optimizer's
+`Optimize()` method is this:
+
+```
+template<typename FunctionType, typename MatType>
+typename MatType::elem_type Optimize(FunctionType& function,
+                                     MatType& coordinates);
+```
+
+The return type, `typename MatType::elem_type`, is just the numeric type held by
+the given matrix type.  So, for `arma::mat`, the return type is just `double`.
+In addition, optimizers for differentiable functions have a third template
+parameter, `GradType`, which specifies the type of the gradient.  `GradType` can
+be manually specified in the situation where, e.g., a sparse gradient is
+desired.
+
+It is easy to write a function to optimize, e.g., an `arma::fmat`.  Here is an
+example, adapted from the `SquaredFunction` example from the
+[arbitrary function documentation](#example__squared_function_optimization).
+
+```c++
+#include <ensmallen.hpp>
+
+class SquaredFunction
+{
+ public:
+  // This returns f(x) = 2 |x|^2.
+  float Evaluate(const arma::fmat& x)
+  {
+    return 2 * std::pow(arma::norm(x), 2.0);
+  }
+
+  void Gradient(const arma::fmat& x, arma::fmat& gradient)
+  {
+    gradient = 4 * x;
+  }
+};
+
+int main()
+{
+  // The minimum is at x = [0 0 0].  Our initial point is chosen to be 
+  // [1.0, -1.0, 1.0].
+  arma::fmat x("1.0 -1.0 1.0");
+
+  // Create simulated annealing optimizer with default options.
+  // The ens::SA<> type can be replaced with any suitable ensmallen optimizer
+  // that is able to handle arbitrary functions.
+  ens::L_BFGS<> optimizer;
+  SquaredFunction f; // Create function to be optimized.
+  optimizer.Optimize(f, x); // The optimizer will infer arma::fmat!
+
+  std::cout << "Minimum of squared function found with simulated annealing is "
+      << x;
+}
+```
+
+Note that we have simply changed the `SquaredFunction` to accept `arma::fmat`
+instead of `arma::mat` as parameters to `Evaluate()`, and the return type has
+accordingly been changed to `float` from `double`.  It would even be possible to
+optimize functions with sparse coordinates by having `Evaluate()` take a sparse
+matrix (i.e. `arma::sp_mat`).
+
+If it were desired to represent the gradient as a sparse type, the `Gradient()`
+function would need to be modified to take a sparse matrix (i.e. `arma::sp_mat`
+or similar), and then you could call `optimizer.Optimize<SquaredFunction,
+arma::mat, arma::sp_mat>(f, x);` to perform the optimization while using sparse
+matrix types to represent the gradient.  Using sparse `MatType` or `GradType`
+should *only* be done when it is known that the objective matrix and/or
+gradients will be sparse; otherwise the code may run very slow!
+
+ensmallen will automatically infer `MatType` from the call to `Optimize()`, and
+check that the given `FunctionType` has all of the necessary functions for the
+given `MatType`, throwing a `static_assert` error if not.  If you would like to
+disable these checks, define the macro `ENS_DISABLE_TYPE_CHECKS` before
+including ensmallen:
+
+```
+#define ENS_DISABLE_TYPE_CHECKS
+#include <ensmallen.hpp>
+```
+
+This can be useful for situations where you know that the checks should be
+ignored.  However, be aware that the code may fail to compile and give more
+confusing and difficult error messages!

doc/optimizers.md

@@ -14,7 +14,7 @@ gradients.
  * `AdaDelta(`_`stepSize`_`)`
  * `AdaDelta(`_`stepSize, batchSize`_`)`
  * `AdaDelta(`_`stepSize, batchSize, rho, epsilon, maxIterations, tolerance, shuffle`_`)`
- * `AdaDelta(`_`stepSize, batchSize, rho, epsilon, maxIterations, tolerance, shuffle, resetPolicy`_`)`
+ * `AdaDelta(`_`stepSize, batchSize, rho, epsilon, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
 
 #### Attributes
 
@@ -28,10 +28,11 @@ gradients.
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 Attributes of the optimizer may also be changed via the member methods
 `StepSize()`, `BatchSize()`, `Rho()`, `Epsilon()`, `MaxIterations()`,
-`Shuffle()`, and `ResetPolicy()`.
+`Shuffle()`, `ResetPolicy()`, and `ExactObjective()`.
 
 #### Examples:
 
@@ -64,8 +65,8 @@ parameters.
  - `AdaGrad(`_`stepSize`_`)`
  - `AdaGrad(`_`stepSize, batchSize`_`)`
  - `AdaGrad(`_`stepSize, batchSize, epsilon, maxIterations, tolerance, shuffle`_`)`
- - `AdaGrad(`_`stepSize, batchSize, epsilon, maxIterations, tolerance, shuffle, resetPolicy`_`)`
- 
+ - `AdaGrad(`_`stepSize, batchSize, epsilon, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
+
 #### Attributes
 
 | **type** | **name** | **description** | **default** |
@@ -77,10 +78,11 @@ parameters.
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `tolerance` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 Attributes of the optimizer may also be changed via the member methods
 `StepSize()`, `BatchSize()`, `Epsilon()`, `MaxIterations()`, `Tolerance()`,
-`Shuffle()`, and `ResetPolicy()`.
+`Shuffle()`, `ResetPolicy()`, and `ExactObjective()`.
 
 #### Examples:
 
@@ -112,7 +114,7 @@ moments.
  * `Adam()`
  * `Adam(`_`stepSize, batchSize`_`)`
  * `Adam(`_`stepSize, batchSize, beta1, beta2, eps, maxIterations, tolerance, shuffle`_`)`
- * `Adam(`_`stepSize, batchSize, beta1, beta2, eps, maxIterations, tolerance, shuffle, resetPolicy`_`)`
+ * `Adam(`_`stepSize, batchSize, beta1, beta2, eps, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
 
 Note that the `Adam` class is based on the `AdamType<`_`UpdateRule`_`>` class
 with _`UpdateRule`_` = AdamUpdate`.
@@ -130,10 +132,11 @@ with _`UpdateRule`_` = AdamUpdate`.
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 The attributes of the optimizer may also be modified via the member methods
 `StepSize()`, `BatchSize()`, `Beta1()`, `Beta2()`, `Eps()`, `MaxIterations()`,
-`Tolerance()`, `Shuffle()`, and `ResetPolicy()`.
+`Tolerance()`, `Shuffle()`, `ResetPolicy()`, and `ExactObjective()`.
 
 #### Examples
 
@@ -163,7 +166,7 @@ AdaMax is simply a variant of Adam based on the infinity norm.
  * `AdaMax()`
  * `AdaMax(`_`stepSize, batchSize`_`)`
  * `AdaMax(`_`stepSize, batchSize, beta1, beta2, eps, maxIterations, tolerance, shuffle`_`)`
- * `AdaMax(`_`stepSize, batchSize, beta1, beta2, eps, maxIterations, tolerance, shuffle, resetPolicy`_`)`
+ * `AdaMax(`_`stepSize, batchSize, beta1, beta2, eps, maxIterations, tolerance, shuffle, exactObjective, resetPolicy`_`)`
 
 Note that the `AdaMax` class is based on the `AdamType<`_`UpdateRule`_`>` class
 with _`UpdateRule`_` = AdaMaxUpdate`.
@@ -180,11 +183,12 @@ with _`UpdateRule`_` = AdaMaxUpdate`.
 | `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
 
 The attributes of the optimizer may also be modified via the member methods
 `StepSize()`, `BatchSize()`, `Beta1()`, `Beta2()`, `Eps()`, `MaxIterations()`,
-`Tolerance()`, `Shuffle()`, and `ResetPolicy()`.
+`Tolerance()`, `Shuffle()`, `ExactObjective()`, and `ResetPolicy()`.
 
 #### Examples
 
@@ -214,7 +218,7 @@ AMSGrad is a variant of Adam with guaranteed convergence.
  * `AMSGrad()`
  * `AMSGrad(`_`stepSize, batchSize`_`)`
  * `AMSGrad(`_`stepSize, batchSize, beta1, beta2, eps, maxIterations, tolerance, shuffle`_`)`
- * `AMSGrad(`_`stepSize, batchSize, beta1, beta2, eps, maxIterations, tolerance, shuffle, resetPolicy`_`)`
+ * `AMSGrad(`_`stepSize, batchSize, beta1, beta2, eps, maxIterations, tolerance, shuffle, exactObjective, resetPolicy`_`)`
 
 Note that the `AMSGrad` class is based on the `AdamType<`_`UpdateRule`_`>` class
 with _`UpdateRule`_` = AMSGradUpdate`.
@@ -231,11 +235,12 @@ with _`UpdateRule`_` = AMSGradUpdate`.
 | `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
 
 The attributes of the optimizer may also be modified via the member methods
 `StepSize()`, `BatchSize()`, `Beta1()`, `Beta2()`, `Eps()`, `MaxIterations()`,
-`Tolerance()`, `Shuffle()`, and `ResetPolicy()`.
+`Tolerance()`, `Shuffle()`, `ExactObjective()`, and `ResetPolicy()`.
 
 #### Examples
 
@@ -344,7 +349,7 @@ sizes without user oversight.
  * `BigBatchSGD<`_`UpdatePolicy`_`>()`
  * `BigBatchSGD<`_`UpdatePolicy`_`>(`_`stepSize`_`)`
  * `BigBatchSGD<`_`UpdatePolicy`_`>(`_`stepSize, batchSize`_`)`
- * `BigBatchSGD<`_`UpdatePolicy`_`>(`_`stepSize, batchSize, epsilon, maxIterations, tolerance, shuffle`_`)`
+ * `BigBatchSGD<`_`UpdatePolicy`_`>(`_`stepSize, batchSize, epsilon, maxIterations, tolerance, shuffle, exactObjective`_`)`
 
 The _`UpdatePolicy`_ template parameter refers to the way that a new step size
 is computed.  The `AdaptiveStepsize` and `BacktrackingLineSearch` classes are
@@ -366,10 +371,11 @@ For convenience the following typedefs have been defined:
 | `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the batch order is shuffled; otherwise, each batch is visited in linear order. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 Attributes of the optimizer may also be changed via the member methods
 `BatchSize()`, `StepSize()`, `BatchDelta()`, `MaxIterations()`, `Tolerance()`,
-and `Shuffle()`.
+`Shuffle()`, and `ExactObjective()`.
 
 #### Examples:
 
@@ -465,14 +471,14 @@ approxOptimizer.Optimize(f, coordinates);
 
 *An optimizer for [arbitrary functions](#arbitrary-functions).*
 
-Conventional Neural Evolution is an optimizer that works like biological evolution which selects best candidates based on their fitness scores and creates new generation by mutation and crossover of population.
+Conventional Neural Evolution is an optimizer that works like biological evolution which selects best candidates based on their fitness scores and creates new generation by mutation and crossover of population. The initial population is generated based on a random normal distribution centered at the given starting point.
 
 #### Constructors
 
  * `CNE()`
  * `CNE(`_`populationSize, maxGenerations`_`)`
  * `CNE(`_`populationSize, maxGenerations, mutationProb, mutationSize`_`)`
- * `CNE(`_`populationSize, maxGenerations, mutationProb, mutationSize, selectPercent, tolerance, objectiveChange`_`)`
+ * `CNE(`_`populationSize, maxGenerations, mutationProb, mutationSize, selectPercent, tolerance`_`)`
 
 #### Attributes
 
@@ -484,11 +490,10 @@ Conventional Neural Evolution is an optimizer that works like biological evoluti
 | `double` | **`mutationSize`** | The range of mutation noise to be added. This range is between 0 and mutationSize. | `0.02` |
 | `double` | **`selectPercent`** | The percentage of candidates to select to become the the next generation. | `0.2` |
 | `double` | **`tolerance`** | The final value of the objective function for termination. If set to negative value, tolerance is not considered. | `1e-5` |
-| `double` | **`objectiveChange`** | Minimum change in best fitness values between two consecutive generations should be greater than threshold. If set to negative value, objectiveChange is not considered. | `1e-5` |
 
 Attributes of the optimizer may also be changed via the member methods
-`PopulationSize()`, `MaxGenerations()`, `MutationProb()`, `SelectPercent()`,
-`Tolerance()`, and `ObjectiveChange()`.
+`PopulationSize()`, `MaxGenerations()`, `MutationProb()`, `SelectPercent()`
+and `Tolerance()`.
 
 #### Examples:
 
@@ -496,16 +501,59 @@ Attributes of the optimizer may also be changed via the member methods
 RosenbrockFunction f;
 arma::mat coordinates = f.GetInitialPoint();
 
-CNE optimizer(200, 10000, 0.2, 0.2, 0.3, 65, 0.1e-4);
+CNE optimizer(200, 10000, 0.2, 0.2, 0.3, 1e-5);
 optimizer.Optimize(f, coordinates);
 ```
 
 #### See also:
 
- * [The CMA Evolution Strategy: A Tutorial](http://www.cmap.polytechnique.fr/~nikolaus.hansen/cmatutorial110628.pdf)
  * [Neuroevolution in Wikipedia](https://en.wikipedia.org/wiki/Neuroevolution)
  * [Arbitrary functions](#arbitrary-functions)
 
+## DE
+
+*An optimizer for [arbitrary functions](#arbitrary-functions).*
+
+Differential Evolution is an evolutionary optimization algorithm which selects best candidates based on their fitness scores and creates new generation by mutation and crossover of population.
+
+#### Constructors
+
+* `DE()`
+* `DE(`_`populationSize, maxGenerations`_`)`
+* `DE(`_`populationSize, maxGenerations, crossoverRate`_`)`
+* `DE(`_`populationSize, maxGenerations, crossoverRate, differentialWeight`_`)`
+* `DE(`_`populationSize, maxGenerations, crossoverRate, differentialWeight, tolerance`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `size_t` | **`populationSize`** | The number of candidates in the population. This should be at least 3 in size. | `100` |
+| `size_t` | **`maxGenerations`** | The maximum number of generations allowed for DE. | `2000` |
+| `double` | **`crossoverRate`** | Probability that a candidate will undergo crossover. | `0.6` |
+| `double` | **`differentialWeight`** | Amplification factor for differentiation. | `0.8` |
+| `double` | **`tolerance`** | The final value of the objective function for termination. If set to negative value, tolerance is not considered. | `1e-5` |
+
+Attributes of the optimizer may also be changed via the member methods
+`PopulationSize()`, `MaxGenerations()`, `CrossoverRate()`, `DifferentialWeight()`
+and `Tolerance()`.
+
+#### Examples:
+
+```c++
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+
+DE optimizer(200, 1000, 0.6, 0.8, 1e-5);
+optimizer.Optimize(f, coordinates);
+```
+
+#### See also:
+
+ * [Differential Evolution - A simple and efficient adaptive scheme for global optimization over continuous spaces](http://www1.icsi.berkeley.edu/~storn/TR-95-012.pdf)
+ * [Differential Evolution in Wikipedia](https://en.wikipedia.org/wiki/Differential_Evolution)
+ * [Arbitrary functions](#arbitrary-functions)
+
 ## Eve
 
 *An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
@@ -532,10 +580,11 @@ Eve is a stochastic gradient based optimization method with locally and globally
 | `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 The attributes of the optimizer may also be modified via the member methods
 `StepSize()`, `BatchSize()`, `Beta1()`, `Beta2()`, `Beta3()`, `Epsilon()`, `Clip()`, `MaxIterations()`,
-`Tolerance()`, and `Shuffle()`.
+`Tolerance()`, `Shuffle()`, and `ExactObjective()`.
 
 #### Examples
 
@@ -619,7 +668,7 @@ changes.
  * `FTML()`
  * `FTML(`_`stepSize, batchSize`_`)`
  * `FTML(`_`stepSize, batchSize, beta1, beta2, epsilon, maxIterations, tolerance, shuffle`_`)`
- * `FTML(`_`stepSize, batchSize, beta1, beta2, epsilon, maxIterations, tolerance, shuffle, resetPolicy`_`)`
+ * `FTML(`_`stepSize, batchSize, beta1, beta2, epsilon, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
 
 #### Attributes
 
@@ -634,10 +683,11 @@ changes.
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 The attributes of the optimizer may also be modified via the member methods
 `StepSize()`, `BatchSize()`, `Beta1()`, `Beta2()`, `Epsilon()`, `MaxIterations()`,
-`Tolerance()`, `Shuffle()`, and `ResetPolicy()`.
+`Tolerance()`, `Shuffle()`, `ResetPolicy()`, and `ExactObjective()`.
 
 #### Examples
 
@@ -838,7 +888,7 @@ update step.
  * `KatyushaType<`_`proximal`_`>()`
  * `KatyushaType<`_`proximal`_`>(`_`convexity, lipschitz`_`)`
  * `KatyushaType<`_`proximal`_`>(`_`convexity, lipschitz, batchSize`_`)`
- * `KatyushaType<`_`proximal`_`>(`_`convexity, lipschitz, batchSize, maxIterations, innerIterations, tolerance, shuffle`_`)`
+ * `KatyushaType<`_`proximal`_`>(`_`convexity, lipschitz, batchSize, maxIterations, innerIterations, tolerance, shuffle, exactObjective`_`)`
 
 The _`proximal`_ template parameter is a boolean value (`true` or `false`) that
 specifies whether or not the proximal update should be used.
@@ -859,10 +909,12 @@ For convenience the following typedefs have been defined:
 | `size_t` | **`innerIterations`** | The number of inner iterations allowed (0 means n / batchSize). Note that the full gradient is only calculated in the outer iteration. | `0` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
+
 
 Attributes of the optimizer may also be changed via the member methods
 `Convexity()`, `Lipschitz()`, `BatchSize()`, `MaxIterations()`,
-`InnerIterations()`, `Tolerance()`, and `Shuffle()`.
+`InnerIterations()`, `Tolerance()`, `Shuffle()`, and `ExactObjective()`.
 
 #### Examples:
 
@@ -889,8 +941,8 @@ proximalOptimizer.Optimize(f, coordinates);
 
 *An optimizer for [differentiable functions](#differentiable-functions)*
 
-L-BFGS is an optimization algorithm in the family of quasi-Newton methods that approximates the Broyden–Fletcher–Goldfarb–Shanno (BFGS) algorithm using a limited amount of computer memory.  
-  
+L-BFGS is an optimization algorithm in the family of quasi-Newton methods that approximates the Broyden-Fletcher-Goldfarb-Shanno (BFGS) algorithm using a limited amount of computer memory.  
+
 #### Constructors
 
  * `L_BFGS()`
@@ -988,7 +1040,7 @@ curves much more steeply (a steep hilly terrain with high curvature).
  * `MomentumSGD()`
  * `MomentumSGD(`_`stepSize, batchSize`_`)`
  * `MomentumSGD(`_`stepSize, batchSize, maxIterations, tolerance, shuffle`_`)`
- * `MomentumSGD(`_`stepSize, batchSize, maxIterations, tolerance, shuffle, momentumPolicy`_`)`
+ * `MomentumSGD(`_`stepSize, batchSize, maxIterations, tolerance, shuffle, momentumPolicy, decayPolicy, resetPolicy, exactObjective`_`)`
 
 Note that `MomentumSGD` is based on the templated type
 `SGD<`_`UpdatePolicyType, DecayPolicyType`_`>` with _`UpdatePolicyType`_` =
@@ -1004,10 +1056,13 @@ MomentumUpdate` and _`DecayPolicyType`_` = NoDecay`.
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `MomentumUpdate` | **`updatePolicy`** | An instantiated `MomentumUpdate`. | `MomentumUpdate()` |
+| `DecayPolicyType` | **`decayPolicy`** | Instantiated decay policy used to adjust the step size. | `DecayPolicyType()` |
+| `bool` | **`resetPolicy`** | Flag that determines whether update policy parameters are reset before every Optimize call. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 Attributes of the optimizer may also be modified via the member methods
-`StepSize()`, `BatchSize()`, `MaxIterations()`, `Tolerance()`, `Shuffle()`, and
-`UpdatePolicy()`.
+`StepSize()`, `BatchSize()`, `MaxIterations()`, `Tolerance()`, `Shuffle()`, `UpdatePolicy()`, `DecayPolicy()`, `ResetPolicy()`, and
+`ExactObjective()`.
 
 Note that the `MomentumUpdate` class has the constructor
 `MomentumUpdate(`_`momentum`_`)` with a default value of `0.5` for the momentum.
@@ -1147,7 +1202,7 @@ the rate of convergence to O(1/k^2).
  * `NesterovMomentumSGD()`
  * `NesterovMomentumSGD(`_`stepSize, batchSize`_`)`
  * `NesterovMomentumSGD(`_`stepSize, batchSize, maxIterations, tolerance, shuffle`_`)`
- * `NesterovMomentumSGD(`_`stepSize, batchSize, maxIterations, tolerance, shuffle, momentumPolicy`_`)`
+ * `NesterovMomentumSGD(`_`stepSize, batchSize, maxIterations, tolerance, shuffle, momentumPolicy, decayPolicy, resetPolicy, exactObjective`_`)`
 
 Note that `MomentumSGD` is based on the templated type
 `SGD<`_`UpdatePolicyType, DecayPolicyType`_`>` with _`UpdatePolicyType`_` =
@@ -1163,10 +1218,13 @@ NesterovMomentumUpdate` and _`DecayPolicyType`_` = NoDecay`.
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `NesterovMomentumUpdate` | **`updatePolicy`** | An instantiated `MomentumUpdate`. | `NesterovMomentumUpdate()` |
+| `DecayPolicyType` | **`decayPolicy`** | Instantiated decay policy used to adjust the step size. | `DecayPolicyType()` |
+| `bool` | **`resetPolicy`** | Flag that determines whether update policy parameters are reset before every Optimize call. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 Attributes of the optimizer may also be modified via the member methods
-`StepSize()`, `BatchSize()`, `MaxIterations()`, `Tolerance()`, `Shuffle()`, and
-`UpdatePolicy()`.
+`StepSize()`, `BatchSize()`, `MaxIterations()`, `Tolerance()`, `Shuffle()`, `UpdatePolicy()`, `DecayPolicy()`, `ResetPolicy()`, and
+`ExactObjective()`.
 
 Note that the `NesterovMomentumUpdate` class has the constructor
 `MomentumUpdate(`_`momentum`_`)` with a default value of `0.5` for the momentum.
@@ -1256,7 +1314,7 @@ Padam is a variant of Adam with a partially adaptive momentum estimation method.
 
  * `Padam()`
  * `Padam(`_`stepSize, batchSize`_`)`
- * `Padam(`_`stepSize, batchSize, beta1, beta2, partial, epsilon, maxIterations, tolerance, shuffle, resetPolicy`_`)`
+ * `Padam(`_`stepSize, batchSize, beta1, beta2, partial, epsilon, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
 
 #### Attributes
 
@@ -1272,10 +1330,11 @@ Padam is a variant of Adam with a partially adaptive momentum estimation method.
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 The attributes of the optimizer may also be modified via the member methods
 `StepSize()`, `BatchSize()`, `Beta1()`, `Beta2()`, `Partial()`, `Epsilon()`,
-`MaxIterations()`, `Tolerance()`, `Shuffle()`, and `ResetPolicy()`.
+`MaxIterations()`, `Tolerance()`, `Shuffle()`, `ResetPolicy()`, and `ExactObjective()`.
 
 #### Examples
 
@@ -1294,6 +1353,84 @@ optimizer.Optimize(f, coordinates);
  * [Adam: A Method for Stochastic Optimization](http://arxiv.org/abs/1412.6980)
  * [Differentiable separable functions](#differentiable-separable-functions)
 
+## PSO
+
+*An optimizer for [arbitrary functions](#arbitrary-functions).*
+
+PSO is an evolutionary approach to optimization that is inspired by flocks or birds or fishes. The fundamental analogy is that every creature (particle in a swarm) is at a measurable position of goodness or fitness, and this information can be shared amongst the creatures in the flock, so that iteratively, the entire flock can get close to the global optimum.
+
+#### Constructors
+
+ * `PSOType<`_`VelocityUpdatePolicy, InitPolicy`_`>()`
+ * `PSOType<`_`VelocityUpdatePolicy, InitPolicy`_`>(`_`numParticles`_`)`
+ * `PSOType<`_`VelocityUpdatePolicy, InitPolicy`_`>(`_`numParticles, lowerBound, upperBound`_`)`
+ * `PSOType<`_`VelocityUpdatePolicy, InitPolicy`_`>(`_`numParticles, lowerBound, upperBound, maxIterations`_`)`
+ * `PSOType<`_`VelocityUpdatePolicy, InitPolicy`_`>(`_`numParticles, lowerBound, upperBound, maxIterations, horizonSize`_`)`
+ * `PSOType<`_`VelocityUpdatePolicy, InitPolicy`_`>(`_`numParticles, lowerBound, upperBound, maxIterations, horizonSize, impTolerance`_`)`
+ * `PSOType<`_`VelocityUpdatePolicy, InitPolicy`_`>(`_`numParticles, lowerBound, upperBound, maxIterations, horizonSize, impTolerance, exploitationFactor, explorationFactor`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `size_t` | **`numParticles`** |  numParticles Number of particles in the swarm. | `64` |
+| `double`, `arma::mat` | **`lowerBound`** | Lower bound of the coordinates of the initial population. | `1` |
+| `double`, `arma::mat` | **`upperBound`** |  Upper bound of the coordinates of the initial population. | `1` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed. | `3000` |
+| `size_t` | **`horizonSize`** | Size of the lookback-horizon for computing improvement. | `350` |
+| `double` | **`impTolerance`** | The final value of the objective function for termination. If set to negative value, tolerance is not considered. | `1e-5` |
+| `double` | **`exploitationFactor`** | Influence of the personal best of the particle. | `2.05` |
+| `double` | **`explorationFactor`** | Influence of the neighbours of the particle. | `2.05` |
+
+Note that the parameters `lowerBound` and `upperBound` are overloaded. Data types of `double` or `arma::mat` may be used. If they are initialized as single values of `double`, then the same value of the bound applies to all the axes, resulting in an initialization following a uniform distribution in a hypercube. If they are initialized as matrices of `arma::mat`, then the value of `lowerBound[i]` applies to axis `[i]`; similarly, for values in `upperBound`. This results in an initialization following a uniform distribution in a hyperrectangle within the specified bounds.
+
+Attributes of the optimizer may also be changed via the member methods
+`NumParticles()`, `LowerBound()`, `UpperBound()`, `MaxIterations()`,
+`HorizonSize()`, `ImpTolerance()`,`ExploitationFactor()`, and
+`ExplorationFactor()`.
+
+At present, only the local-best variant of PSO is present in ensmallen. The optimizer may be initialized using the class type `LBestPSO`, which is an alias for `PSOType<LBestUpdate, DefaultInit>`.
+
+#### Examples:
+
+```c++
+SphereFunction f(4);
+arma::vec coordinates = f.GetInitialPoint();
+
+LBestPSO s;
+const double result = s.Optimize(f, coordinates)
+```
+
+```c++
+RosenbrockFunction f;
+arma::vec coordinates = f.GetInitialPoint();
+
+// Setting bounds for the initial swarm population of size 2.
+arma::vec lowerBound("50 50");
+arma::vec upperBound("60 60");
+
+LBestPSO s(200, lowerBound, upperBound, 3000, 600, 1e-30, 2.05, 2.05);
+const double result = s.Optimize(f, coordinates)
+```
+
+```c++
+RosenbrockFunction f;
+arma::vec coordinates = f.GetInitialPoint();
+
+// Setting bounds for the initial swarm population as type double.
+double lowerBound = 50;
+double upperBound = 60;
+
+LBestPSO s(64, lowerBound, upperBound, 3000, 400, 1e-30, 2.05, 2.05);
+const double result = s.Optimize(f, coordinates)
+```
+
+#### See also:
+
+ * [Particle Swarm Optimization](http://www.swarmintelligence.org/)
+ * [Arbitrary functions](#arbitrary-functions)
+
+
 ## Primal-dual SDP Solver
 
 *An optimizer for [semidefinite programs](#semidefinite-programs).*
@@ -1303,14 +1440,8 @@ programs.
 
 #### Constructors
 
- * `PrimalDualSolver<`_`SDPType`_`>(`_`sdp`_`)`
- * `PrimalDualSolver<`_`SDPType`_`>(`_`sdp, initialX, initialYSparse, initialYDense, initialZ`_`)`
-
-The _`SDPType`_ template parameter specifies the type of SDP to solve.  The
-`SDP<arma::mat>` and `SDP<arma::sp_mat>` classes are available for use; these
-represent SDPs with dense and sparse `C` matrices, respectively.  The `SDP<>`
-class is detailed in the [semidefinite program
-documentation](#semidefinite-programs).
+ * `PrimalDualSolver<>(`_`maxIterations`_`)`
+ * `PrimalDualSolver<>(`_`maxIterations, tau, normXzTol, primalInfeasTol, dualInfeasTol`_`)`
 
 #### Attributes
 
@@ -1335,7 +1466,9 @@ optionally return the converged values for the dual variables.
  * Invoke the optimization procedure, returning the converged values for the
  * primal and dual variables.
  */
-double Optimize(arma::mat& X,
+template<typename SDPType>
+double Optimize(SDPType& s,
+                arma::mat& X,
                 arma::vec& ySparse,
                 arma::vec& yDense,
                 arma::mat& Z);
@@ -1343,15 +1476,144 @@ double Optimize(arma::mat& X,
 /**
  * Invoke the optimization procedure, and only return the primal variable.
  */
-double Optimize(arma::mat& X);
+template<typename SDPType>
+double Optimize(SDPType& s, arma::mat& X);
 ```
 
+The _`SDPType`_ template parameter specifies the type of SDP to solve.  The
+`SDP<arma::mat>` and `SDP<arma::sp_mat>` classes are available for use; these
+represent SDPs with dense and sparse `C` matrices, respectively.  The `SDP<>`
+class is detailed in the [semidefinite program
+documentation](#semidefinite-programs).  _`SDPType`_ is automatically inferred
+when `Optimize()` is called with an SDP.
+
 #### See also:
 
  * [Primal-dual interior-point methods for semidefinite programming](http://www.dtic.mil/dtic/tr/fulltext/u2/1020236.pdf)
  * [Semidefinite programming on Wikipedia](https://en.wikipedia.org/wiki/Semidefinite_programming)
  * [Semidefinite programs](#semidefinite-programs) (includes example usage of `PrimalDualSolver`)
 
+## Quasi-Hyperbolic Momentum Update SGD (QHSGD)
+
+*An optimizer for [differentiable separable
+functions](#differentiable-separable-functions).*
+
+Quasi-hyperbolic momentum update SGD (QHSGD) is an SGD-like optimizer with
+momentum where quasi-hyperbolic terms are added to the parametrization.  The
+update rule for this optimizer is a weighted average of momentum SGD and vanilla
+SGD.
+
+#### Constructors
+
+  * `QHSGD()`
+  * `QHSGD(`_`stepSize, batchSize`_`)`
+  * `QHSGD(`_`stepSize, batchSize, maxIterations, tolerance, shuffle, exactObjective`_`)`
+
+ Note that `QHSGD` is based on the templated type
+ `SGD<`_`UpdatePolicyType, DecayPolicyType`_`>` with _`UpdatePolicyType`_` =
+ QHUpdate` and _`DecayPolicyType`_` = NoDecay`.
+
+#### Attributes
+
+ | **type** | **name** | **description** | **default** |
+ |----------|----------|-----------------|-------------|
+ | `double` | **`stepSize`** | Step size for each iteration. | `0.01` |
+ | `size_t` | **`batchSize`** | Batch size to use for each step. | `32` |
+ | `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+ | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
+ | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
+ | `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
+
+ Attributes of the optimizer may also be modified via the member methods
+ `StepSize()`, `BatchSize()`, `MaxIterations()`, `Tolerance()`, `Shuffle()`,  and `ExactObjective()`.
+
+ Note that the `QHUpdate` class has the constructor  `QHUpdate(`_`v,
+momentum`_`)` with a default value of `0.7` for the quasi-hyperbolic term `v`
+and `0.999` for the momentum term.
+
+#### Examples
+
+ ```c++
+ RosenbrockFunction f;
+ arma::mat coordinates = f.GetInitialPoint();
+
+ QHSGD optimizer(0.01, 32, 100000, 1e-5, true);
+ optimizer.Optimize(f, coordinates);
+ ```
+
+#### See also:
+
+  * [Quasi-Hyperbolic Momentum and Adam For Deep Learning](https://arxiv.org/pdf/1810.06801.pdf)
+  * [SGD](#sgd)
+  * [Momentum SGD](#momentum-sgd)
+  * [Nesterov Momentum SGD](#nesterov-momentum-sgd)
+  * [SGD in Wikipedia](https://en.wikipedia.org/wiki/Stochastic_gradient_descent)
+  * [Differentiable separable functions](#differentiable-separable-functions)
+
+## QHAdam
+
+*An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
+
+QHAdam is an optimizer that uses quasi-hyperbolic descent with the Adam
+optimizer.  This replaces the moment estimators of Adam with quasi-hyperbolic
+terms, and different values of the `v1` and `v2` parameters are equivalent to
+the following other optimizers:
+
+ * When `v1 = v2 = 1`, `QHAdam` is equivalent to `Adam`.
+
+ * When `v1 = 0` and `v2 = 1`, `QHAdam` is equivalent to `RMSProp`.
+
+ * When `v1 = beta1` and `v2 = 1`, `QHAdam` is equivalent to `Nadam`.
+
+#### Constructors
+
+  * `QHAdam()`
+  * `QHAdam(`_`stepSize, batchSize`_`)`
+  * `QHAdam(`_`stepSize, batchSize, v1, v2, beta1, beta2, eps, maxIterations`_`)`
+  * `QHAdam(`_`stepSize, batchSize, v1, v2, beta1, beta2, eps, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
+
+#### Attributes
+
+ | **type** | **name** | **description** | **default** |
+ |----------|----------|-----------------|-------------|
+ | `double` | **`stepSize`** | Step size for each iteration. | `0.001` |
+ | `size_t` | **`batchSize`** | Number of points to process in a single step. | `32` |
+ | `double` | **`v1`** | The First Quasi Hyperbolic Term. | `0.7` |
+ | `double` | **`v2`** | The Second Quasi Hyperbolic Term. | `1.00` |
+ | `double` | **`beta1`** | Exponential decay rate for the first moment estimates. | `0.9` |
+ | `double` | **`beta2`** | Exponential decay rate for the weighted infinity norm estimates. | `0.999` |
+ | `double` | **`eps`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
+ | `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+ | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
+ | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
+ | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+ | `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
+
+ The attributes of the optimizer may also be modified via the member methods
+ `StepSize()`, `BatchSize()`, `Beta1()`, `Beta2()`, `Eps()`, `MaxIterations()`,
+ `Tolerance()`, `Shuffle()`, `V1()`, `V2()`, `ResetPolicy()`, and `ExactObjective()`.
+
+#### Examples
+
+ ```c++
+ RosenbrockFunction f;
+ arma::mat coordinates = f.GetInitialPoint();
+
+ QHAdam optimizer(0.001, 32, 0.7, 0.9, 0.9, 0.999, 1e-8, 100000, 1e-5, true);
+ optimizer.Optimize(f, coordinates);
+ ```
+
+#### See also:
+
+  * [Quasi-Hyperbolic Momentum and Adam For Deep Learning](https://arxiv.org/pdf/1810.06801.pdf)
+  * [SGD in Wikipedia](https://en.wikipedia.org/wiki/Stochastic_gradient_descent)
+  * [SGD](#standard-sgd)
+  * [Adam](#adam)
+  * [RMSprop](#rmsprop)
+  * [Nadam](#nadam)
+  * [Incorporating Nesterov Momentum into Adam](http://cs229.stanford.edu/proj2015/054_report.pdf)
+  * [Differentiable separable functions](#differentiable-separable-functions)
+
 ## RMSProp
 
 *An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
@@ -1363,7 +1625,7 @@ RMSProp utilizes the magnitude of recent gradients to normalize the gradients.
  * `RMSProp()`
  * `RMSProp(`_`stepSize, batchSize`_`)`
  * `RMSProp(`_`stepSize, batchSize, alpha, epsilon, maxIterations, tolerance, shuffle`_`)`
- * `RMSProp(`_`stepSize, batchSize, alpha, epsilon, maxIterations, tolerance, shuffle, resetPolicy`_`)`
+ * `RMSProp(`_`stepSize, batchSize, alpha, epsilon, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
 
 #### Attributes
 
@@ -1377,10 +1639,11 @@ RMSProp utilizes the magnitude of recent gradients to normalize the gradients.
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 Attributes of the optimizer can also be modified via the member methods
 `StepSize()`, `BatchSize()`, `Alpha()`, `Epsilon()`, `MaxIterations()`,
-`Tolerance()`, `Shuffle()`, and `ResetPolicy()`.
+`Tolerance()`, `Shuffle()`, `ResetPolicy()`, and `ExactObjective()`.
 
 #### Examples:
 
@@ -1469,30 +1732,28 @@ optimizer.Optimize(f, coordinates);
 
 ## Simultaneous Perturbation Stochastic Approximation (SPSA)
 
-*An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
+*An optimizer for [arbitrary functions](#arbitrary-functions).*
 
 The SPSA algorithm approximates the gradient of the function by finite
 differences along stochastic directions.
 
-### Constructors
+#### Constructors
 
- * `SPSA(`_`alpha, batchSize, gamma, stepSize, evaluationStepSize, maxIterations, tolerance, shuffle`_`)`
+ * `SPSA(`_`alpha, gamma, stepSize, evaluationStepSize, maxIterations, tolerance`_`)`
 
- #### Attributes
+#### Attributes
 
 | **type** | **name** | **description** | **default** |
 |----------|----------|-----------------|-------------|
 | `double` | **`alpha`** | Scaling exponent for the step size. | `0.602` |
-| `size_t` | **`batchSize`** | Batch size to use for each step. | `32` |
 | `double` | **`gamma`** | Scaling exponent for evaluation step size. | `0.101` |
 | `double` | **`stepSize`** | Scaling parameter for step size (named as 'a' in the paper). | `0.16` |
 | `double` | **`evaluationStepSize`** | Scaling parameter for evaluation step size (named as 'c' in the paper). | `0.3` |
 | `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
-| `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 
 Attributes of the optimizer may also be changed via the member methods
-`Alpha()`, `BatchSize()`, `Gamma()`, `StepSize()`, `EvaluationStepSize()`, and `MaxIterations()`.
+`Alpha()`, `Gamma()`, `StepSize()`, `EvaluationStepSize()`, and `MaxIterations()`.
 
 #### Examples:
 
@@ -1500,7 +1761,7 @@ Attributes of the optimizer may also be changed via the member methods
 SphereFunction f(2);
 arma::mat coordinates = f.GetInitialPoint();
 
-SPSA optimizer(0.1, 2, 0.102, 0.16, 0.3, 100000, 0);
+SPSA optimizer(0.1, 0.102, 0.16, 0.3, 100000, 1e-5);
 optimizer.Optimize(f, coordinates);
 ```
 
@@ -1524,7 +1785,7 @@ losses.
 
  * `SARAHType<`_`UpdatePolicyType`_`>()`
  * `SARAHType<`_`UpdatePolicyType`_`>(`_`stepSize, batchSize`_`)`
- * `SARAHType<`_`UpdatePolicyType`_`>(`_`stepSize, batchSize, maxIterations, innerIterations, tolerance, shuffle, updatePolicy`_`)`
+ * `SARAHType<`_`UpdatePolicyType`_`>(`_`stepSize, batchSize, maxIterations, innerIterations, tolerance, shuffle, updatePolicy, exactObjective`_`)`
 
 The _`UpdatePolicyType`_ template parameter specifies the update step used for
 the optimizer.  The `SARAHUpdate` and `SARAHPlusUpdate` classes are available
@@ -1548,10 +1809,11 @@ For convenience the following typedefs have been defined:
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `UpdatePolicyType` | **`updatePolicy`** | Instantiated update policy used to adjust the given parameters. | `UpdatePolicyType()` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 Attributes of the optimizer may also be changed via the member methods
 `StepSize()`, `BatchSize()`, `MaxIterations()`, `InnerIterations()`,
-`Tolerance()`, `Shuffle()`, and `UpdatePolicy()`.
+`Tolerance()`, `Shuffle()`, `UpdatePolicy()`, and `ExactObjective()`.
 
 Note that the default value for `updatePolicy` is the default constructor for
 the `UpdatePolicyType`.
@@ -1590,7 +1852,7 @@ implementation may still be useful in some situations.
 
  * `StandardSGD()`
  * `StandardSGD(`_`stepSize, batchSize`_`)`
- * `StandardSGD(`_`stepSize, batchSize, maxIterations, tolerance, shuffle`_`)`
+ * `StandardSGD(`_`stepSize, batchSize, maxIterations, tolerance, shuffle, updatePolicy, decayPolicy, resetPolicy, exactObjective`_`)`
 
 Note that `StandardSGD` is based on the templated type
 `SGD<`_`UpdatePolicyType, DecayPolicyType`_`>` with _`UpdatePolicyType`_` =
@@ -1605,9 +1867,14 @@ VanillaUpdate` and _`DecayPolicyType`_` = NoDecay`.
 | `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
+| `UpdatePolicyType` | **`updatePolicy`** | Instantiated update policy used to adjust the given parameters. | `UpdatePolicyType()` |
+| `DecayPolicyType` | **`decayPolicy`** | Instantiated decay policy used to adjust the step size. | `DecayPolicyType()` |
+| `bool` | **`resetPolicy`** | Flag that determines whether update policy parameters are reset before every Optimize call. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 Attributes of the optimizer may also be modified via the member methods
-`StepSize()`, `BatchSize()`, `MaxIterations()`, `Tolerance()`, and `Shuffle()`.
+`StepSize()`, `BatchSize()`, `MaxIterations()`, `Tolerance()`, `Shuffle()`, `UpdatePolicy()`, `DecayPolicy()`, `ResetPolicy()`, and
+`ExactObjective()`.
 
 #### Examples
 
@@ -1695,16 +1962,18 @@ cyclicscd.Optimize(f, coordinates);
 
 ## Stochastic Gradient Descent with Restarts (SGDR)
 
-*An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
+*An optimizer for [differentiable separable
+functions](#differentiable-separable-functions).*
 
-SGDR is based on Mini-batch Stochastic Gradient Descent class and simulates a new warm-started run/restart once a number of epochs are performed. 
+SGDR is based on Mini-batch Stochastic Gradient Descent class and simulates a
+new warm-started run/restart once a number of epochs are performed.
 
 #### Constructors
 
  * `SGDR<`_`UpdatePolicyType`_`>()`
  * `SGDR<`_`UpdatePolicyType`_`>(`_`epochRestart, multFactor, batchSize, stepSize`_`)`
  * `SGDR<`_`UpdatePolicyType`_`>(`_`epochRestart, multFactor, batchSize, stepSize, maxIterations, tolerance, shuffle, updatePolicy`_`)`
- * `SGDR<`_`UpdatePolicyType`_`>(`_`epochRestart, multFactor, batchSize, stepSize, maxIterations, tolerance, shuffle, updatePolicy`_`, resetPolicy)`
+ * `SGDR<`_`UpdatePolicyType`_`>(`_`epochRestart, multFactor, batchSize, stepSize, maxIterations, tolerance, shuffle, updatePolicy, resetPolicy, exactObjective`_`)`
 
 The _`UpdatePolicyType`_ template parameter controls the update policy used
 during the iterative update process.  The `MomentumUpdate` class is available
@@ -1728,11 +1997,12 @@ so the shorter type `SGDR<>` can be used instead of the equivalent
 | `bool` | **`shuffle`** | If true, the mini-batch order is shuffled; otherwise, each mini-batch is visited in linear order. | `true` |
 | `UpdatePolicyType` | **`updatePolicy`** | Instantiated update policy used to adjust the given parameters. | `UpdatePolicyType()` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 Attributes of the optimizer can also be modified via the member methods
 `EpochRestart()`, `MultFactor()`, `BatchSize()`, `StepSize()`,
-`MaxIterations()`, `Tolerance()`, `Shuffle()`, `UpdatePolicy()`, and
-`ResetPolicy()`.
+`MaxIterations()`, `Tolerance()`, `Shuffle()`, `UpdatePolicy()`, `ResetPolicy()`, and
+`ExactObjective()`.
 
 Note that the default value for `updatePolicy` is the default constructor for
 the `UpdatePolicyType`.
@@ -1765,7 +2035,7 @@ are performed using the Snapshot Ensembles technique.
  * `SnapshotSGDR<`_`UpdatePolicyType`_`>()`
  * `SnapshotSGDR<`_`UpdatePolicyType`_`>(`_`epochRestart, multFactor, batchSize, stepSize`_`)`
  * `SnapshotSGDR<`_`UpdatePolicyType`_`>(`_`epochRestart, multFactor, batchSize, stepSize, maxIterations, tolerance, shuffle, snapshots, accumulate, updatePolicy`_`)`
- * `SnapshotSGDR<`_`UpdatePolicyType`_`>(`_`epochRestart, multFactor, batchSize, stepSize, maxIterations, tolerance, shuffle, snapshots, accumulate, updatePolicy, resetPolicy`_`)`
+ * `SnapshotSGDR<`_`UpdatePolicyType`_`>(`_`epochRestart, multFactor, batchSize, stepSize, maxIterations, tolerance, shuffle, snapshots, accumulate, updatePolicy, resetPolicy, exactObjective`_`)`
 
 The _`UpdatePolicyType`_ template parameter controls the update policy used
 during the iterative update process.  The `MomentumUpdate` class is available
@@ -1791,11 +2061,12 @@ so the shorter type `SnapshotSGDR<>` can be used instead of the equivalent
 | `bool` | **`accumulate`** | Accumulate the snapshot parameter. | `true` |
 | `UpdatePolicyType` | **`updatePolicy`** | Instantiated update policy used to adjust the given parameters. | `UpdatePolicyType()` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 Attributes of the optimizer can also be modified via the member methods
 `EpochRestart()`, `MultFactor()`, `BatchSize()`, `StepSize()`,
 `MaxIterations()`, `Tolerance()`, `Shuffle()`, `Snapshots()`, `Accumulate()`,
-`UpdatePolicy()`, and `ResetPolicy()`.
+`UpdatePolicy()`, `ResetPolicy()`, and `ExactObjective()`.
 
 The `Snapshots()` function returns a `std::vector<arma::mat>&` (a vector of
 snapshots of the parameters), not a `size_t` representing the maximum number of
@@ -1834,7 +2105,7 @@ parameter space.
  * `SMORMS3()`
  * `SMORMS3(`_`stepSize, batchSize`_`)`
  * `SMORMS3(`_`stepSize, batchSize, epsilon, maxIterations, tolerance`_`)`
- * `SMORMS3(`_`stepSize, batchSize, epsilon, maxIterations, tolerance, shuffle, resetPolicy`_`)`
+ * `SMORMS3(`_`stepSize, batchSize, epsilon, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
 
 #### Attributes
 
@@ -1847,10 +2118,11 @@ parameter space.
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the mini-batch order is shuffled; otherwise, each mini-batch is visited in linear order. | `true` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 Attributes of the optimizer can also be modified via the member methods
 `StepSize()`, `BatchSize()`, `Epsilon()`, `MaxIterations()`, `Tolerance()`,
-`Shuffle()`, and `ResetPolicy()`.
+`Shuffle()`, `ResetPolicy()`, and `ExactObjective()`.
 
 #### Examples:
 
@@ -1881,7 +2153,7 @@ strongly convex problems.
  * `SVRGType<`_`UpdatePolicyType, DecayPolicyType`_`>()`
  * `SVRGType<`_`UpdatePolicyType, DecayPolicyType`_`>(`_`stepSize`_`)`
  * `SVRGType<`_`UpdatePolicyType, DecayPolicyType`_`>(`_`stepSize, batchSize, maxIterations, innerIterations`_`)`
- * `SVRGType<`_`UpdatePolicyType, DecayPolicyType`_`>(`_`stepSize, batchSize, maxIterations, innerIterations, tolerance, shuffle, updatePolicy, decayPolicy, resetPolicy`_`)`
+ * `SVRGType<`_`UpdatePolicyType, DecayPolicyType`_`>(`_`stepSize, batchSize, maxIterations, innerIterations, tolerance, shuffle, updatePolicy, decayPolicy, resetPolicy, exactObjective`_`)`
 
 The _`UpdatePolicyType`_ template parameter controls the update step used by
 SVRG during the optimization.  The `SVRGUpdate` class is available for use and
@@ -1911,11 +2183,12 @@ For convenience the following typedefs have been defined:
 | `UpdatePolicyType` | **`updatePolicy`** | Instantiated update policy used to adjust the given parameters. | `UpdatePolicyType()` |
 | `DecayPolicyType` | **`decayPolicy`** | Instantiated decay policy used to adjust the step size. | `DecayPolicyType()` |
 | `bool` | **`resetPolicy`** | Flag that determines whether update policy parameters are reset before every Optimize call. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 Attributes of the optimizer may also be modified via the member methods
 `StepSize()`, `BatchSize()`, `MaxIterations()`, `InnerIterations()`,
-`Tolerance()`, `Shuffle()`, `UpdatePolicy()`, `DecayPolicy()`, and
-`ResetPolicy()`.
+`Tolerance()`, `Shuffle()`, `UpdatePolicy()`, `DecayPolicy()`, `ResetPolicy()`, and
+`ExactObjective()`.
 
 Note that the default values for the `updatePolicy` and `decayPolicy` parameters
 are simply the default constructors of the _`UpdatePolicyType`_ and
@@ -1958,7 +2231,7 @@ the system back on track.
  * `SPALeRASGD<`_`DecayPolicyType`_`>()`
  * `SPALeRASGD<`_`DecayPolicyType`_`>(`_`stepSize, batchSize`_`)`
  * `SPALeRASGD<`_`DecayPolicyType`_`>(`_`stepSize, batchSize, maxIterations, tolerance`_`)`
- * `SPALeRASGD<`_`DecayPolicyType`_`>(`_`stepSize, batchSize, maxIterations, tolerance, lambda, alpha, epsilon, adaptRate, shuffle, decayPolicy, resetPolicy`_`)`
+ * `SPALeRASGD<`_`DecayPolicyType`_`>(`_`stepSize, batchSize, maxIterations, tolerance, lambda, alpha, epsilon, adaptRate, shuffle, decayPolicy, resetPolicy, exactObjective`_`)`
 
 The _`DecayPolicyType`_ template parameter controls the decay in the step size
 during the course of the optimization.  The `NoDecay` class is available for
@@ -1983,11 +2256,11 @@ By default, _`DecayPolicyType`_ is set to `NoDecay`, so the shorter type
 | `bool` | **`shuffle`** | If true, the batch order is shuffled; otherwise, each batch is visited in linear order. | `true` |
 | `DecayPolicyType` | **`decayPolicy`** | Instantiated decay policy used to adjust the step size. | `DecayPolicyType()` |
 | `bool` | **`resetPolicy`** | Flag that determines whether update policy parameters are reset before every Optimize call. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 Attributes of the optimizer may also be modified via the member methods
 `StepSize()`, `BatchSize()`, `MaxIterations()`, `Tolerance()`, `Lambda()`,
-`Alpha()`, `Epsilon()`, `AdaptRate()`, `Shuffle()`, `DecayPolicy()`, and
-`ResetPolicy()`.
+`Alpha()`, `Epsilon()`, `AdaptRate()`, `Shuffle()`, `DecayPolicy()`, `ResetPolicy()`, and `ExactObjective()`.
 
 #### Examples
 
@@ -2019,6 +2292,7 @@ the projection of Adam steps on the gradient subspace.
  * `SWATS()`
  * `SWATS(`_`stepSize, batchSize`_`)`
  * `SWATS(`_`stepSize, batchSize, beta1, beta2, epsilon, maxIterations, tolerance`_`)`
+ * `SWATS(`_`stepSize, batchSize, beta1, beta2, epsilon, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
 
 #### Attributes
 
@@ -2032,10 +2306,12 @@ the projection of Adam steps on the gradient subspace.
 | `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the mini-batch order is shuffled; otherwise, each mini-batch is visited in linear order. | `true` |
+| `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 Attributes of the optimizer can also be modified via the member methods
 `StepSize()`, `BatchSize()`, `Beta1()`, `Beta2()`, `Epsilon()`,
-`MaxIterations()`, `Tolerance()`, and `Shuffle()`.
+`MaxIterations()`, `Tolerance()`, `Shuffle()`, `ResetPolicy()`, and  `ExactObjective()`.
 
 #### Examples:
 
@@ -2067,7 +2343,7 @@ near-optimal convergence rates in both the batch and stochastic settings.
  * `WNGrad()`
  * `WNGrad(`_`stepSize, batchSize`_`)`
  * `WNGrad(`_`stepSize, batchSize, maxIterations, tolerance, shuffle`_`)`
- * `WNGrad(`_`stepSize, batchSize, maxIterations, tolerance, shuffle, resetPolicy`_`)`
+ * `WNGrad(`_`stepSize, batchSize, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
 
 #### Attributes
 
@@ -2079,10 +2355,11 @@ near-optimal convergence rates in both the batch and stochastic settings.
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the batch order is shuffled; otherwise, each batch is visited in linear order. | `true` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 Attributes of the optimizer may also be modified via the member methods
-`StepSize()`, `BatchSize()`, `MaxIterations()`, `Tolerance()`, `Shuffle()`, and
-`ResetPolicy()`.
+`StepSize()`, `BatchSize()`, `MaxIterations()`, `Tolerance()`, `Shuffle()`, `ResetPolicy()`, and
+`ExactObjective()`.
 
 #### Examples
 

include/ensmallen.hpp

@@ -59,15 +59,28 @@
 #include "ensmallen_bits/ens_version.hpp"
 #include "ensmallen_bits/log.hpp" // TODO: should move to another place
 
+#include "ensmallen_bits/utility/any.hpp"
+#include "ensmallen_bits/utility/arma_traits.hpp"
+
+// Callbacks.
+#include "ensmallen_bits/callbacks/callbacks.hpp"
+#include "ensmallen_bits/callbacks/early_stop_at_min_loss.hpp"
+#include "ensmallen_bits/callbacks/print_loss.hpp"
+#include "ensmallen_bits/callbacks/progress_bar.hpp"
+#include "ensmallen_bits/callbacks/store_best_coordinates.hpp"
+#include "ensmallen_bits/callbacks/timer_stop.hpp"
+
 #include "ensmallen_bits/problems/problems.hpp" // TODO: should move to another place
 
 #include "ensmallen_bits/ada_delta/ada_delta.hpp"
 #include "ensmallen_bits/ada_grad/ada_grad.hpp"
 #include "ensmallen_bits/adam/adam.hpp"
+#include "ensmallen_bits/qhadam/qhadam.hpp"
 #include "ensmallen_bits/aug_lagrangian/aug_lagrangian.hpp"
 #include "ensmallen_bits/bigbatch_sgd/bigbatch_sgd.hpp"
 #include "ensmallen_bits/cmaes/cmaes.hpp"
 #include "ensmallen_bits/cne/cne.hpp"
+#include "ensmallen_bits/de/de.hpp"
 #include "ensmallen_bits/eve/eve.hpp"
 #include "ensmallen_bits/ftml/ftml.hpp"
 
@@ -81,6 +94,7 @@
 #include "ensmallen_bits/lbfgs/lbfgs.hpp"
 #include "ensmallen_bits/padam/padam.hpp"
 #include "ensmallen_bits/parallel_sgd/parallel_sgd.hpp"
+#include "ensmallen_bits/pso/pso.hpp"
 #include "ensmallen_bits/rmsprop/rmsprop.hpp"
 
 #include "ensmallen_bits/sa/sa.hpp"

include/ensmallen_bits/ada_delta/ada_delta.hpp

@@ -66,6 +66,8 @@ class AdaDelta
    *        function is visited in linear order.
    * @param resetPolicy If true, parameters are reset before every Optimize
    *        call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
    */
   AdaDelta(const double stepSize = 1.0,
            const size_t batchSize = 32,
@@ -74,7 +76,8 @@ class AdaDelta
            const size_t maxIterations = 100000,
            const double tolerance = 1e-5,
            const bool shuffle = true,
-           const bool resetPolicy = true);
+           const bool resetPolicy = true,
+           const bool exactObjective = false);
 
   /**
    * Optimize the given function using AdaDelta. The given starting point will
@@ -82,15 +85,40 @@ class AdaDelta
    * objective value is returned. The DecomposableFunctionType is checked for
    * API consistency at compile time.
    *
-   * @tparam DecomposableFunctionType Type of the function to optimize.
+   * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate)
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(DecomposableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
   {
-    return optimizer.Optimize(function, iterate);
+    return optimizer.Optimize<DecomposableFunctionType, MatType, GradType,
+        CallbackTypes...>(function, iterate, callbacks...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(DecomposableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<DecomposableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
   }
 
   //! Get the step size.
@@ -128,6 +156,11 @@ class AdaDelta
   //! Modify whether or not the individual functions are shuffled.
   bool& Shuffle() { return optimizer.Shuffle(); }
 
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
   //! Get whether or not the update policy parameters
   //! are reset before Optimize call.
   bool ResetPolicy() const { return optimizer.ResetPolicy(); }

include/ensmallen_bits/ada_delta/ada_delta_impl.hpp

@@ -26,7 +26,8 @@ inline AdaDelta::AdaDelta(const double stepSize,
                           const size_t maxIterations,
                           const double tolerance,
                           const bool shuffle,
-                          const bool resetPolicy) :
+                          const bool resetPolicy,
+                          const bool exactObjective) :
     optimizer(stepSize,
               batchSize,
               maxIterations,
@@ -34,7 +35,8 @@ inline AdaDelta::AdaDelta(const double stepSize,
               shuffle,
               AdaDeltaUpdate(rho, epsilon),
               NoDecay(),
-              resetPolicy)
+              resetPolicy,
+              exactObjective)
 { /* Nothing to do. */ }
 
 } // namespace ens

include/ensmallen_bits/ada_delta/ada_delta_update.hpp

@@ -51,49 +51,6 @@ class AdaDeltaUpdate
     // Nothing to do.
   }
 
-  /**
-   * The Initialize method is called by SGD Optimizer method before the start of
-   * the iteration update process. In AdaDelta update policy, the mean squared
-   * and the delta mean squared gradient matrices are initialized to the zeros
-   * matrix with the same size as gradient matrix (see ens::SGD<>).
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    // Initialize empty matrices for mean sum of squares of parameter gradient.
-    meanSquaredGradient = arma::zeros<arma::mat>(rows, cols);
-    meanSquaredGradientDx = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for SGD. The AdaDelta update dynamically adapts over time using
-   * only first order information. Additionally, AdaDelta requires no manual
-   * tuning of a learning rate.
-   *
-   * @param iterate Parameters that minimize the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    // Accumulate gradient.
-    meanSquaredGradient *= rho;
-    meanSquaredGradient += (1 - rho) * (gradient % gradient);
-    arma::mat dx = arma::sqrt((meanSquaredGradientDx + epsilon) /
-        (meanSquaredGradient + epsilon)) % gradient;
-
-    // Accumulate updates.
-    meanSquaredGradientDx *= rho;
-    meanSquaredGradientDx += (1 - rho) * (dx % dx);
-
-    // Apply update.
-    iterate -= (stepSize * dx);
-  }
-
   //! Get the smoothing parameter.
   double Rho() const { return rho; }
   //! Modify the smoothing parameter.
@@ -104,18 +61,77 @@ class AdaDeltaUpdate
   //! Modify the value used to initialise the mean squared gradient parameter.
   double& Epsilon() { return epsilon; }
 
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD optimizer method before the start
+     * of the iteration update process. In AdaDelta update policy, the mean
+     * squared and the delta mean squared gradient matrices are initialized to
+     * the zeros matrix with the same size as gradient matrix (see ens::SGD<>).
+     *
+     * @param parent AdaDeltaUpdate object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AdaDeltaUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent)
+    {
+      meanSquaredGradient.zeros(rows, cols);
+      meanSquaredGradientDx.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for SGD. The AdaDelta update dynamically adapts over time
+     * using only first order information. Additionally, AdaDelta requires no
+     * manual tuning of a learning rate.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Accumulate gradient.
+      meanSquaredGradient *= parent.rho;
+      meanSquaredGradient += (1 - parent.rho) * (gradient % gradient);
+      GradType dx = arma::sqrt((meanSquaredGradientDx + parent.epsilon) /
+          (meanSquaredGradient + parent.epsilon)) % gradient;
+
+      // Accumulate updates.
+      meanSquaredGradientDx *= parent.rho;
+      meanSquaredGradientDx += (1 - parent.rho) * (dx % dx);
+
+      // Apply update.
+      iterate -= (stepSize * dx);
+    }
+
+   private:
+    // The instantiated parent class.
+    AdaDeltaUpdate& parent;
+
+    // The mean squared gradient matrix.
+    GradType meanSquaredGradient;
+
+    // The delta mean squared gradient matrix.
+    GradType meanSquaredGradientDx;
+  };
+
  private:
   // The smoothing parameter.
   double rho;
 
   // The epsilon value used to initialise the mean squared gradient parameter.
   double epsilon;
-
-  // The mean squared gradient matrix.
-  arma::mat meanSquaredGradient;
-
-  // The delta mean squared gradient matrix.
-  arma::mat meanSquaredGradientDx;
 };
 
 } // namespace ens

include/ensmallen_bits/ada_grad/ada_grad.hpp

@@ -64,6 +64,8 @@ class AdaGrad
    *        function is visited in linear order.
    * @param resetPolicy If true, parameters are reset before every Optimize
    *        call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
    */
   AdaGrad(const double stepSize = 0.01,
           const size_t batchSize = 32,
@@ -71,22 +73,48 @@ class AdaGrad
           const size_t maxIterations = 100000,
           const double tolerance = 1e-5,
           const bool shuffle = true,
-          const bool resetPolicy = true);
+          const bool resetPolicy = true,
+          const bool exactObjective = false);
 
   /**
    * Optimize the given function using AdaGrad. The given starting point will
    * be modified to store the finishing point of the algorithm, and the final
    * objective value is returned.
    *
-   * @tparam DecomposableFunctionType Type of the function to optimize.
+   * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate)
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(DecomposableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
   {
-    return optimizer.Optimize(function, iterate);
+    return optimizer.Optimize<DecomposableFunctionType, MatType, GradType,
+        CallbackTypes...>(function, iterate, callbacks...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(DecomposableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<DecomposableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
   }
 
   //! Get the step size.
@@ -119,6 +147,11 @@ class AdaGrad
   //! Modify whether or not the individual functions are shuffled.
   bool& Shuffle() { return optimizer.Shuffle(); }
 
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
   //! Get whether or not the update policy parameters
   //! are reset before Optimize call.
   bool ResetPolicy() const { return optimizer.ResetPolicy(); }

include/ensmallen_bits/ada_grad/ada_grad_impl.hpp

@@ -23,7 +23,8 @@ inline AdaGrad::AdaGrad(const double stepSize,
                         const size_t maxIterations,
                         const double tolerance,
                         const bool shuffle,
-                        const bool resetPolicy) :
+                        const bool resetPolicy,
+                        const bool exactObjective) :
     optimizer(stepSize,
               batchSize,
               maxIterations,
@@ -31,7 +32,8 @@ inline AdaGrad::AdaGrad(const double stepSize,
               shuffle,
               AdaGradUpdate(epsilon),
               NoDecay(),
-              resetPolicy)
+              resetPolicy,
+              exactObjective)
 { /* Nothing to do. */ }
 
 } // namespace ens

include/ensmallen_bits/ada_grad/ada_grad_update.hpp

@@ -49,49 +49,67 @@ class AdaGradUpdate
     // Nothing to do.
   }
 
-  /**
-   * The Initialize method is called by SGD Optimizer method before the start of
-   * the iteration update process. In AdaGrad update policy, squared
-   * gradient matrix is initialized to the zeros matrix with the same size as
-   * gradient matrix (see ens::SGD<>).
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    // Initialize an empty matrix for sum of squares of parameter gradient.
-    squaredGradient = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for SGD. The AdaGrad update adapts the learning rate by
-   * performing larger updates for more sparse parameters and smaller updates
-   * for less sparse parameters .
-   *
-   * @param iterate Parameters that minimize the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    squaredGradient += (gradient % gradient);
-    iterate -= (stepSize * gradient) / (arma::sqrt(squaredGradient) + epsilon);
-  }
-
   //! Get the value used to initialise the squared gradient parameter.
   double Epsilon() const { return epsilon; }
   //! Modify the value used to initialise the squared gradient parameter.
   double& Epsilon() { return epsilon; }
 
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD optimizer before the start of the
+     * iteration update process. In AdaGrad update policy, squared gradient
+     * matrix is initialized to the zeros matrix with the same size as gradient
+     * matrix (see ens::SGD<>).
+     *
+     * @param parent Instantiated parent class.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AdaGradUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent),
+        squaredGradient(rows, cols)
+    {
+      // Initialize an empty matrix for sum of squares of parameter gradient.
+      squaredGradient.zeros();
+    }
+
+    /**
+     * Update step for SGD. The AdaGrad update adapts the learning rate by
+     * performing larger updates for more sparse parameters and smaller updates
+     * for less sparse parameters.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      squaredGradient += (gradient % gradient);
+      iterate -= (stepSize * gradient) / (arma::sqrt(squaredGradient) +
+          parent.epsilon);
+    }
+
+   private:
+    // Instantiated parent class.
+    AdaGradUpdate& parent;
+    // The squared gradient matrix.
+    GradType squaredGradient;
+  };
+
  private:
   // The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
-
-  // The squared gradient matrix.
-  arma::mat squaredGradient;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/adam.hpp

@@ -11,7 +11,7 @@
  * for first-order gradient-based optimization of stochastic objective
  * functions, based on adaptive estimates of lower-order moments. AdaMax is
  * simply a variant of Adam based on the infinity norm. AMSGrad is another
- * variant of Adam with guaranteed convergence. Nadam is another variant of 
+ * variant of Adam with guaranteed convergence. Nadam is another variant of
  * Adam based on NAG. NadaMax is a variant for Nadam based on Infinity form.
  *
  * ensmallen is free software; you may redistribute it and/or modify it under
@@ -88,6 +88,8 @@ class AdamType
    *        function is visited in linear order.
    * @param resetPolicy If true, parameters are reset before every Optimize
    *        call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
    */
   AdamType(const double stepSize = 0.001,
            const size_t batchSize = 32,
@@ -97,22 +99,49 @@ class AdamType
            const size_t maxIterations = 100000,
            const double tolerance = 1e-5,
            const bool shuffle = true,
-           const bool resetPolicy = true);
+           const bool resetPolicy = true,
+           const bool exactObjective = false);
 
   /**
    * Optimize the given function using Adam. The given starting point will be
    * modified to store the finishing point of the algorithm, and the final
    * objective value is returned.
    *
-   * @tparam DecomposableFunctionType Type of the function to optimize.
+   * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate)
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(DecomposableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
   {
-    return optimizer.Optimize(function, iterate);
+    return optimizer.template Optimize<
+        DecomposableFunctionType, MatType, GradType, CallbackTypes...>(
+        function, iterate, callbacks...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(DecomposableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<DecomposableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
   }
 
   //! Get the step size.
@@ -155,6 +184,11 @@ class AdamType
   //! Modify whether or not the individual functions are shuffled.
   bool& Shuffle() { return optimizer.Shuffle(); }
 
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
   //! Get whether or not the update policy parameters
   //! are reset before Optimize call.
   bool ResetPolicy() const { return optimizer.ResetPolicy(); }

include/ensmallen_bits/adam/adam_impl.hpp

@@ -30,7 +30,8 @@ AdamType<UpdateRule>::AdamType(
     const size_t maxIterations,
     const double tolerance,
     const bool shuffle,
-    const bool resetPolicy) :
+    const bool resetPolicy,
+    const bool exactObjective) :
     optimizer(stepSize,
               batchSize,
               maxIterations,
@@ -38,7 +39,8 @@ AdamType<UpdateRule>::AdamType(
               shuffle,
               UpdateRule(epsilon, beta1, beta2),
               NoDecay(),
-              resetPolicy)
+              resetPolicy,
+              exactObjective)
 { /* Nothing to do. */ }
 
 } // namespace ens

include/ensmallen_bits/adam/adam_update.hpp

@@ -58,52 +58,6 @@ class AdamUpdate
     // Nothing to do.
   }
 
-  /**
-   * The Initialize method is called by SGD Optimizer method before the start of
-   * the iteration update process.
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    m = arma::zeros<arma::mat>(rows, cols);
-    v = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for Adam.
-   *
-   * @param iterate Parameters that minimize the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    // Increment the iteration counter variable.
-    ++iteration;
-
-    // And update the iterate.
-    m *= beta1;
-    m += (1 - beta1) * gradient;
-
-    v *= beta2;
-    v += (1 - beta2) * (gradient % gradient);
-
-    const double biasCorrection1 = 1.0 - std::pow(beta1, iteration);
-    const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
-
-    /**
-     * It should be noted that the term, m / (arma::sqrt(v) + eps), in the
-     * following expression is an approximation of the following actual term;
-     * m / (arma::sqrt(v) + (arma::sqrt(biasCorrection2) * eps).
-     */
-    iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
-        m / (arma::sqrt(v) + epsilon);
-  }
-
   //! Get the value used to initialise the squared gradient parameter.
   double Epsilon() const { return epsilon; }
   //! Modify the value used to initialise the squared gradient parameter.
@@ -119,6 +73,82 @@ class AdamUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent AdamUpdate object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AdamUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent)
+    {
+      m.zeros(rows, cols);
+      v.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for Adam.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // And update the iterate.
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      v *= parent.beta2;
+      v += (1 - parent.beta2) * (gradient % gradient);
+
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
+          parent.iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
+          parent.iteration);
+
+      /**
+       * It should be noted that the term, m / (arma::sqrt(v) + eps), in the
+       * following expression is an approximation of the following actual term;
+       * m / (arma::sqrt(v) + (arma::sqrt(biasCorrection2) * eps).
+       */
+      iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
+          m / (arma::sqrt(v) + parent.epsilon);
+    }
+
+   private:
+    // Instantiated parent object.
+    AdamUpdate& parent;
+
+    // The exponential moving average of gradient values.
+    GradType m;
+
+    // The exponential moving average of squared gradient values.
+    GradType v;
+  };
+
  private:
   // The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
@@ -129,14 +159,8 @@ class AdamUpdate
   // The second moment coefficient.
   double beta2;
 
-  // The exponential moving average of gradient values.
-  arma::mat m;
-
-  // The exponential moving average of squared gradient values.
-  arma::mat v;
-
   // The number of iterations.
-  double iteration;
+  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/adamax_update.hpp

@@ -60,47 +60,6 @@ class AdaMaxUpdate
     // Nothing to do.
   }
 
-  /**
-   * The Initialize method is called by SGD Optimizer method before the start of
-   * the iteration update process.
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    m = arma::zeros<arma::mat>(rows, cols);
-    u = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for Adam.
-   *
-   * @param iterate Parameters that minimize the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    // Increment the iteration counter variable.
-    ++iteration;
-
-    // And update the iterate.
-    m *= beta1;
-    m += (1 - beta1) * gradient;
-
-    // Update the exponentially weighted infinity norm.
-    u *= beta2;
-    u = arma::max(u, arma::abs(gradient));
-
-    const double biasCorrection1 = 1.0 - std::pow(beta1, iteration);
-
-    if (biasCorrection1 != 0)
-      iterate -= (stepSize / biasCorrection1 * m / (u + epsilon));
-  }
-
   //! Get the value used to initialise the squared gradient parameter.
   double Epsilon() const { return epsilon; }
   //! Modify the value used to initialise the squared gradient parameter.
@@ -116,6 +75,74 @@ class AdaMaxUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent AdaMaxUpdate object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AdaMaxUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent)
+    {
+      m.zeros(rows, cols);
+      u.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for AdaMax.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // And update the iterate.
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      // Update the exponentially weighted infinity norm.
+      u *= parent.beta2;
+      u = arma::max(u, arma::abs(gradient));
+
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
+          parent.iteration);
+
+      if (biasCorrection1 != 0)
+        iterate -= (stepSize / biasCorrection1 * m / (u + parent.epsilon));
+    }
+
+   private:
+    // Instantiated parent object.
+    AdaMaxUpdate& parent;
+    // The exponential moving average of gradient values.
+    GradType m;
+    // The exponentially weighted infinity norm.
+    GradType u;
+  };
+
  private:
   // The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
@@ -126,14 +153,8 @@ class AdaMaxUpdate
   // The second moment coefficient.
   double beta2;
 
-  // The exponential moving average of gradient values.
-  arma::mat m;
-
-  // The exponentially weighted infinity norm.
-  arma::mat u;
-
   // The number of iterations.
-  double iteration;
+  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/amsgrad_update.hpp

@@ -53,51 +53,6 @@ class AMSGradUpdate
     // Nothing to do.
   }
 
-  /**
-   * The Initialize method is called by SGD Optimizer method before the start of
-   * the iteration update process.
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    m = arma::zeros<arma::mat>(rows, cols);
-    v = arma::zeros<arma::mat>(rows, cols);
-    vImproved = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for AMSGrad.
-   *
-   * @param iterate Parameters that minimize the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    // Increment the iteration counter variable.
-    ++iteration;
-
-    // And update the iterate.
-    m *= beta1;
-    m += (1 - beta1) * gradient;
-
-    v *= beta2;
-    v += (1 - beta2) * (gradient % gradient);
-
-    const double biasCorrection1 = 1.0 - std::pow(beta1, iteration);
-    const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
-
-    // Element wise maximum of past and present squared gradients.
-    vImproved = arma::max(vImproved, v);
-
-    iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
-                m / (arma::sqrt(vImproved) + epsilon);
-  }
-
   //! Get the value used to initialise the squared gradient parameter.
   double Epsilon() const { return epsilon; }
   //! Modify the value used to initialise the squared gradient parameter.
@@ -113,6 +68,84 @@ class AMSGradUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent Instantiated AMSGradUpdate parent object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AMSGradUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent)
+    {
+      m.zeros(rows, cols);
+      v.zeros(rows, cols);
+      vImproved.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for AMSGrad.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // And update the iterate.
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      v *= parent.beta2;
+      v += (1 - parent.beta2) * (gradient % gradient);
+
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
+          parent.iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
+          parent.iteration);
+
+      // Element wise maximum of past and present squared gradients.
+      vImproved = arma::max(vImproved, v);
+
+      iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
+                  m / (arma::sqrt(vImproved) + parent.epsilon);
+    }
+
+   private:
+    // Instantiated parent AMSGradUpdate object.
+    AMSGradUpdate& parent;
+
+    // The exponential moving average of gradient values.
+    GradType m;
+
+    // The exponential moving average of squared gradient values.
+    GradType v;
+
+    // The optimal squared gradient value.
+    GradType vImproved;
+  };
+
  private:
   // The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
@@ -123,17 +156,8 @@ class AMSGradUpdate
   // The second moment coefficient.
   double beta2;
 
-  // The exponential moving average of gradient values.
-  arma::mat m;
-
-  // The exponential moving average of squared gradient values.
-  arma::mat v;
-
-  // The optimal sqaured gradient value.
-  arma::mat vImproved;
-
   // The number of iterations.
-  double iteration;
+  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/nadam_update.hpp

@@ -51,78 +51,16 @@ class NadamUpdate
       beta1(beta1),
       beta2(beta2),
       scheduleDecay(scheduleDecay),
-      iteration(0),
-      cumBeta1(1)
+      iteration(0)
   {
     // Nothing to do.
   }
 
-  /**
-   * The Initialize() method is called by the optimizer before the start of the
-   * iteration update process.
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    m = arma::zeros<arma::mat>(rows, cols);
-    v = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for Nadam.
-   *
-   * @param iterate Parameters that minimize the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    // Increment the iteration counter variable.
-    ++iteration;
-
-    // And update the iterate.
-    m *= beta1;
-    m += (1 - beta1) * gradient;
-
-    v *= beta2;
-    v += (1 - beta2) * gradient % gradient;
-
-    double beta1T = beta1 * (1 - (0.5 *
-        std::pow(0.96, iteration * scheduleDecay)));
-
-    double beta1T1 = beta1 * (1 - (0.5 *
-        std::pow(0.96, (iteration + 1) * scheduleDecay)));
-
-    cumBeta1 *= beta1T;
-
-    const double biasCorrection1 = 1.0 - cumBeta1;
-
-    const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
-
-    const double biasCorrection3 = 1.0 - (cumBeta1 * beta1T1);
-
-    /* Note :- arma::sqrt(v) + epsilon * sqrt(biasCorrection2) is approximated
-     * as arma::sqrt(v) + epsilon
-     */
-    iterate -= (stepSize * (((1 - beta1T) / biasCorrection1) * gradient
-        + (beta1T1 / biasCorrection3) * m) * sqrt(biasCorrection2))
-        / (arma::sqrt(v) + epsilon);
-  }
-
   //! Get the value used to initialise the squared gradient parameter.
   double Epsilon() const { return epsilon; }
   //! Modify the value used to initialise the squared gradient parameter.
   double& Epsilon() { return epsilon; }
 
-  //! Get the value of the cumulative product of decay coefficients
-  double CumBeta1() const { return cumBeta1; }
-  //! Modify the value of the cumulative product of decay coefficients
-  double& CumBeta1() { return cumBeta1; }
-
   //! Get the smoothing parameter.
   double Beta1() const { return beta1; }
   //! Modify the smoothing parameter.
@@ -138,6 +76,95 @@ class NadamUpdate
   //! Modify the decay parameter for decay coefficients
   double& ScheduleDecay() { return scheduleDecay; }
 
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the optimizer before the start of the
+     * iteration update process.
+     *
+     * @param parent Instantiated NadamUpdate parent object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(NadamUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent),
+        cumBeta1(1)
+    {
+      m.zeros(rows, cols);
+      v.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for Nadam.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // And update the iterate.
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      v *= parent.beta2;
+      v += (1 - parent.beta2) * gradient % gradient;
+
+      double beta1T = parent.beta1 * (1 - (0.5 *
+          std::pow(0.96, parent.iteration * parent.scheduleDecay)));
+
+      double beta1T1 = parent.beta1 * (1 - (0.5 *
+          std::pow(0.96, (parent.iteration + 1) * parent.scheduleDecay)));
+
+      cumBeta1 *= beta1T;
+
+      const double biasCorrection1 = 1.0 - cumBeta1;
+
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
+          parent.iteration);
+
+      const double biasCorrection3 = 1.0 - (cumBeta1 * beta1T1);
+
+      /* Note :- arma::sqrt(v) + epsilon * sqrt(biasCorrection2) is approximated
+       * as arma::sqrt(v) + epsilon
+       */
+      iterate -= (stepSize * (((1 - beta1T) / biasCorrection1) * gradient
+          + (beta1T1 / biasCorrection3) * m) * sqrt(biasCorrection2))
+          / (arma::sqrt(v) + parent.epsilon);
+    }
+
+   private:
+    // Instantiated parent object.
+    NadamUpdate& parent;
+
+    // The exponential moving average of gradient values.
+    GradType m;
+
+    // The exponential moving average of squared gradient values.
+    GradType v;
+
+    // The cumulative product of decay coefficients.
+    double cumBeta1;
+  };
+
  private:
   // The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
@@ -148,20 +175,11 @@ class NadamUpdate
   // The second moment coefficient.
   double beta2;
 
-  // The exponential moving average of gradient values.
-  arma::mat m;
-
-  // The exponential moving average of squared gradient values.
-  arma::mat v;
-
-  // The decay parameter for decay coefficients
+  // The decay parameter for decay coefficients.
   double scheduleDecay;
 
   // The number of iterations.
-  double iteration;
-
-  // The cumulative product of decay coefficients
-  double cumBeta1;
+  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/nadamax_update.hpp

@@ -51,74 +51,16 @@ class NadaMaxUpdate
       beta1(beta1),
       beta2(beta2),
       scheduleDecay(scheduleDecay),
-      cumBeta1(1),
       iteration(0)
   {
     // Nothing to do.
   }
 
-  /**
-   * The Initialize() method is called by the optimizer before the start of the
-   * iteration update process.
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    m = arma::zeros<arma::mat>(rows, cols);
-    u = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for NadaMax.
-   *
-   * @param iterate Parameters that minimize the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    // Increment the iteration counter variable.
-    ++iteration;
-
-    // And update the iterate.
-    m *= beta1;
-    m += (1 - beta1) * gradient;
-
-    u = arma::max(u * beta2, arma::abs(gradient));
-
-    double beta1T = beta1 * (1 - (0.5 *
-        std::pow(0.96, iteration * scheduleDecay)));
-
-    double beta1T1 = beta1 * (1 - (0.5 *
-        std::pow(0.96, (iteration + 1) * scheduleDecay)));
-
-    cumBeta1 *= beta1T;
-
-    const double biasCorrection1 = 1.0 - cumBeta1;
-
-    const double biasCorrection2 = 1.0 - (cumBeta1 * beta1T1);
-
-    if ((biasCorrection1 != 0) && (biasCorrection2 != 0))
-    {
-       iterate -= (stepSize * (((1 - beta1T) / biasCorrection1) * gradient
-           + (beta1T1 / biasCorrection2) * m)) / (u + epsilon);
-    }
-  }
-
   //! Get the value used to initialise the squared gradient parameter.
   double Epsilon() const { return epsilon; }
   //! Modify the value used to initialise the squared gradient parameter.
   double& Epsilon() { return epsilon; }
 
-  //! Get the value of the cumulative product of decay coefficients
-  double CumBeta1() const { return cumBeta1; }
-  //! Modify the value of the cumulative product of decay coefficients
-  double& CumBeta1() { return cumBeta1; }
-
   //! Get the smoothing parameter.
   double Beta1() const { return beta1; }
   //! Modify the smoothing parameter.
@@ -134,6 +76,90 @@ class NadaMaxUpdate
   //! Modify the decay parameter for decay coefficients
   double& ScheduleDecay() { return scheduleDecay; }
 
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor method is called by the optimizer before the start of
+     * the iteration update process.
+     *
+     * @param parent Instantiated NadaMaxUpdate parent object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(NadaMaxUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent),
+        cumBeta1(1)
+    {
+      m.zeros(rows, cols);
+      u.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for NadaMax.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // And update the iterate.
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      u = arma::max(u * parent.beta2, arma::abs(gradient));
+
+      double beta1T = parent.beta1 * (1 - (0.5 *
+          std::pow(0.96, parent.iteration * parent.scheduleDecay)));
+
+      double beta1T1 = parent.beta1 * (1 - (0.5 *
+          std::pow(0.96, (parent.iteration + 1) * parent.scheduleDecay)));
+
+      cumBeta1 *= beta1T;
+
+      const double biasCorrection1 = 1.0 - cumBeta1;
+
+      const double biasCorrection2 = 1.0 - (cumBeta1 * beta1T1);
+
+      if ((biasCorrection1 != 0) && (biasCorrection2 != 0))
+      {
+         iterate -= (stepSize * (((1 - beta1T) / biasCorrection1) * gradient
+             + (beta1T1 / biasCorrection2) * m)) / (u + parent.epsilon);
+      }
+    }
+
+   private:
+    // Instantiated parent object.
+    NadaMaxUpdate& parent;
+
+    // The exponential moving average of gradient values.
+    GradType m;
+
+    // The exponentially weighted infinity norm.
+    GradType u;
+
+    // The cumulative product of decay coefficients.
+    double cumBeta1;
+  };
+
  private:
   // The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
@@ -144,20 +170,11 @@ class NadaMaxUpdate
   // The second moment coefficient.
   double beta2;
 
-  // The exponential moving average of gradient values.
-  arma::mat m;
-
-  // The exponentially weighted infinity norm.
-  arma::mat u;
-
-  // The decay parameter for decay coefficients
+  // The decay parameter for decay coefficients.
   double scheduleDecay;
 
-  // The cumulative product of decay coefficients
-  double cumBeta1;
-
   // The number of iterations.
-  double iteration;
+  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/optimisticadam_update.hpp

@@ -57,51 +57,6 @@ class OptimisticAdamUpdate
     // Nothing to do.
   }
 
-  /**
-   * The Initialize method is called by SGD Optimizer method before the start of
-   * the iteration update process.
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    m = arma::zeros<arma::mat>(rows, cols);
-    v = arma::zeros<arma::mat>(rows, cols);
-    g = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for OptimisticAdam.
-   *
-   * @param iterate Parameters that minimize the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    // Increment the iteration counter variable.
-    ++iteration;
-
-    // And update the iterate.
-    m *= beta1;
-    m += (1 - beta1) * gradient;
-
-    v *= beta2;
-    v += (1 - beta2) * arma::square(gradient);
-
-    arma::mat mCorrected = m / (1.0 - std::pow(beta1, iteration));
-    arma::mat vCorrected = v / (1.0 - std::pow(beta2, iteration));
-
-    arma::mat update = mCorrected / (arma::sqrt(vCorrected) + epsilon);
-
-    iterate -= (2 * stepSize * update - stepSize * g);
-
-    g = std::move(update);
-  }
-
   //! Get the value used to initialize the squared gradient parameter.
   double Epsilon() const { return epsilon; }
   //! Modify the value used to initialize the squared gradient parameter.
@@ -117,6 +72,85 @@ class OptimisticAdamUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent Instantiated OptimisticAdamUpdate parent object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(OptimisticAdamUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent)
+    {
+      m.zeros(rows, cols);
+      v.zeros(rows, cols);
+      g.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for OptimisticAdam.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // And update the iterate.
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      v *= parent.beta2;
+      v += (1 - parent.beta2) * arma::square(gradient);
+
+      GradType mCorrected = m / (1.0 - std::pow(parent.beta1,
+          parent.iteration));
+      GradType vCorrected = v / (1.0 - std::pow(parent.beta2,
+          parent.iteration));
+
+      GradType update = mCorrected /
+          (arma::sqrt(vCorrected) + parent.epsilon);
+
+      iterate -= (2 * stepSize * update - stepSize * g);
+
+      g = std::move(update);
+    }
+
+   private:
+    // Instantiated parent object.
+    OptimisticAdamUpdate& parent;
+
+    // The exponential moving average of gradient values.
+    GradType m;
+
+    // The exponential moving average of squared gradient values.
+    GradType v;
+
+    // The previous update.
+    GradType g;
+  };
+
  private:
   // The epsilon value used to initialize the squared gradient parameter.
   double epsilon;
@@ -127,16 +161,8 @@ class OptimisticAdamUpdate
   // The second moment coefficient.
   double beta2;
 
-  // The exponential moving average of gradient values.
-  arma::mat m;
-
-  // The exponential moving average of squared gradient values.
-  arma::mat v;
-  // The previous update.
-  arma::mat g;
-
   // The number of iterations.
-  double iteration;
+  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/aug_lagrangian/aug_lagrangian.hpp

@@ -55,12 +55,34 @@ class AugLagrangian
    *
    * @tparam LagrangianFunctionType Function which can be optimized by this
    *     class.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function The function to optimize.
    * @param coordinates Output matrix to store the optimized coordinates in.
+   * @param callbacks Callback functions.
    */
-  template<typename LagrangianFunctionType>
+  template<typename LagrangianFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value, bool>::type
+  Optimize(LagrangianFunctionType& function,
+           MatType& coordinates,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename LagrangianFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
   bool Optimize(LagrangianFunctionType& function,
-                arma::mat& coordinates);
+                MatType& coordinates,
+                CallbackTypes&&... callbacks)
+  {
+    return Optimize<LagrangianFunctionType, MatType, MatType,
+        CallbackTypes...>(function, coordinates,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
 
   /**
    * Optimize the function, giving initial estimates for the Lagrange
@@ -69,17 +91,41 @@ class AugLagrangian
    *
    * @tparam LagrangianFunctionType Function which can be optimized by this
    *      class.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function The function to optimize.
    * @param coordinates Output matrix to store the optimized coordinates in.
    * @param initLambda Vector of initial Lagrange multipliers.  Should have
    *     length equal to the number of constraints.
    * @param initSigma Initial penalty parameter.
+   * @param callbacks Callback functions.
    */
-  template<typename LagrangianFunctionType>
+  template<typename LagrangianFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value, bool>::type
+  Optimize(LagrangianFunctionType& function,
+           MatType& coordinates,
+           const arma::vec& initLambda,
+           const double initSigma,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename LagrangianFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
   bool Optimize(LagrangianFunctionType& function,
-                arma::mat& coordinates,
+                MatType& coordinates,
                 const arma::vec& initLambda,
-                const double initSigma);
+                const double initSigma,
+                CallbackTypes&&... callbacks)
+  {
+    return Optimize<LagrangianFunctionType, MatType, MatType,
+        CallbackTypes...>(function, coordinates, initLambda, initSigma,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
 
   //! Get the L-BFGS object used for the actual optimization.
   const L_BFGS& LBFGS() const { return lbfgs; }
@@ -124,8 +170,12 @@ class AugLagrangian
   //! The L-BFGS optimizer that we will use.
   L_BFGS lbfgs;
 
+  //! Controls early termination of the optimization process.
+  bool terminate;
+
   //! Lagrange multipliers.
   arma::vec lambda;
+
   //! Penalty parameter.
   double sigma;
 
@@ -133,9 +183,27 @@ class AugLagrangian
    * Internal optimization function: given an initialized AugLagrangianFunction,
    * perform the optimization itself.
    */
-  template<typename LagrangianFunctionType>
-  bool Optimize(AugLagrangianFunction<LagrangianFunctionType>& augfunc,
-                arma::mat& coordinates);
+  template<typename LagrangianFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value, bool>::type
+  Optimize(AugLagrangianFunction<LagrangianFunctionType>& augfunc,
+           MatType& coordinates,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename LagrangianFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  bool Optimize(AugLagrangianFunction<LagrangianFunctionType>& function,
+                MatType& coordinates,
+                CallbackTypes&&... callbacks)
+  {
+    return Optimize<LagrangianFunctionType, MatType, MatType,
+        CallbackTypes...>(function, coordinates,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
 };
 
 } // namespace ens

include/ensmallen_bits/aug_lagrangian/aug_lagrangian_function.hpp

@@ -63,7 +63,8 @@ class AugLagrangianFunction
    * @param coordinates Coordinates to evaluate function at.
    * @return Objective function.
    */
-  double Evaluate(const arma::mat& coordinates) const;
+  template<typename MatType>
+  typename MatType::elem_type Evaluate(const MatType& coordinates) const;
 
   /**
    * Evaluate the gradient of the Augmented Lagrangian function.
@@ -71,7 +72,8 @@ class AugLagrangianFunction
    * @param coordinates Coordinates to evaluate gradient at.
    * @param gradient Matrix to store gradient into.
    */
-  void Gradient(const arma::mat& coordinates, arma::mat& gradient) const;
+  template<typename MatType, typename GradType>
+  void Gradient(const MatType& coordinates, GradType& gradient) const;
 
   /**
    * Get the initial point of the optimization (supplied by the
@@ -79,7 +81,8 @@ class AugLagrangianFunction
    *
    * @return Initial point.
    */
-  const arma::mat& GetInitialPoint() const;
+  template<typename MatType = arma::mat>
+  const MatType& GetInitialPoint() const;
 
   //! Get the Lagrange multipliers.
   const arma::vec& Lambda() const { return lambda; }

include/ensmallen_bits/aug_lagrangian/aug_lagrangian_function_impl.hpp

@@ -46,19 +46,22 @@ AugLagrangianFunction<LagrangianFunction>::AugLagrangianFunction(
 
 // Evaluate the AugLagrangianFunction at the given coordinates.
 template<typename LagrangianFunction>
-double AugLagrangianFunction<LagrangianFunction>::Evaluate(
-    const arma::mat& coordinates) const
+template<typename MatType>
+typename MatType::elem_type AugLagrangianFunction<LagrangianFunction>::Evaluate(
+    const MatType& coordinates) const
 {
   // The augmented Lagrangian is evaluated as
   //    f(x) + {-lambda_i * c_i(x) + (sigma / 2) c_i(x)^2} for all constraints
 
+  typedef typename MatType::elem_type ElemType;
+
   // First get the function's objective value.
-  double objective = function.Evaluate(coordinates);
+  ElemType objective = function.Evaluate(coordinates);
 
   // Now loop for each constraint.
   for (size_t i = 0; i < function.NumConstraints(); ++i)
   {
-    double constraint = function.EvaluateConstraint(i, coordinates);
+    ElemType constraint = function.EvaluateConstraint(i, coordinates);
 
     objective += (-lambda[i] * constraint) +
         sigma * std::pow(constraint, 2) / 2;
@@ -69,22 +72,23 @@ double AugLagrangianFunction<LagrangianFunction>::Evaluate(
 
 // Evaluate the gradient of the AugLagrangianFunction at the given coordinates.
 template<typename LagrangianFunction>
+template<typename MatType, typename GradType>
 void AugLagrangianFunction<LagrangianFunction>::Gradient(
-    const arma::mat& coordinates,
-    arma::mat& gradient) const
+    const MatType& coordinates,
+    GradType& gradient) const
 {
   // The augmented Lagrangian's gradient is evaluted as
   // f'(x) + {(-lambda_i + sigma * c_i(x)) * c'_i(x)} for all constraints
   gradient.zeros();
   function.Gradient(coordinates, gradient);
 
-  arma::mat constraintGradient; // Temporary for constraint gradients.
+  GradType constraintGradient; // Temporary for constraint gradients.
   for (size_t i = 0; i < function.NumConstraints(); i++)
   {
     function.GradientConstraint(i, coordinates, constraintGradient);
 
     // Now calculate scaling factor and add to existing gradient.
-    arma::mat tmpGradient;
+    GradType tmpGradient;
     tmpGradient = (-lambda[i] + sigma *
         function.EvaluateConstraint(i, coordinates)) * constraintGradient;
     gradient += tmpGradient;
@@ -93,10 +97,11 @@ void AugLagrangianFunction<LagrangianFunction>::Gradient(
 
 // Get the initial point.
 template<typename LagrangianFunction>
-const arma::mat& AugLagrangianFunction<LagrangianFunction>::GetInitialPoint()
+template<typename MatType>
+const MatType& AugLagrangianFunction<LagrangianFunction>::GetInitialPoint()
     const
 {
-  return function.GetInitialPoint();
+  return function.template GetInitialPoint<MatType>();
 }
 
 } // namespace ens

include/ensmallen_bits/aug_lagrangian/aug_lagrangian_impl.hpp

@@ -26,15 +26,21 @@ inline AugLagrangian::AugLagrangian(const size_t maxIterations,
     maxIterations(maxIterations),
     penaltyThresholdFactor(penaltyThresholdFactor),
     sigmaUpdateFactor(sigmaUpdateFactor),
-    lbfgs(lbfgs)
+    lbfgs(lbfgs),
+    terminate(false)
 {
 }
 
-template<typename LagrangianFunctionType>
-bool AugLagrangian::Optimize(LagrangianFunctionType& function,
-                             arma::mat& coordinates,
-                             const arma::vec& initLambda,
-                             const double initSigma)
+template<typename LagrangianFunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value, bool>::type
+AugLagrangian::Optimize(LagrangianFunctionType& function,
+                        MatType& coordinates,
+                        const arma::vec& initLambda,
+                        const double initSigma,
+                        CallbackTypes&&... callbacks)
 {
   lambda = initLambda;
   sigma = initSigma;
@@ -42,46 +48,75 @@ bool AugLagrangian::Optimize(LagrangianFunctionType& function,
   AugLagrangianFunction<LagrangianFunctionType> augfunc(function,
       lambda, sigma);
 
-  return Optimize(augfunc, coordinates);
+  return Optimize(augfunc, coordinates, callbacks...);
 }
 
-template<typename LagrangianFunctionType>
-bool AugLagrangian::Optimize(LagrangianFunctionType& function,
-                             arma::mat& coordinates)
+template<typename LagrangianFunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value, bool>::type
+AugLagrangian::Optimize(LagrangianFunctionType& function,
+                        MatType& coordinates,
+                        CallbackTypes&&... callbacks)
 {
   // If the user did not specify the right size for sigma and lambda, we will
   // use defaults.
   if (!lambda.is_empty())
   {
-    AugLagrangianFunction<LagrangianFunctionType> augfunc(function, lambda, sigma);
-    return Optimize(augfunc, coordinates);
+    AugLagrangianFunction<LagrangianFunctionType> augfunc(function, lambda,
+        sigma);
+    return Optimize(augfunc, coordinates, callbacks...);
   }
   else
   {
     AugLagrangianFunction<LagrangianFunctionType> augfunc(function);
-    return Optimize(augfunc, coordinates);
+    return Optimize(augfunc, coordinates, callbacks...);
   }
 }
 
-template<typename LagrangianFunctionType>
-bool AugLagrangian::Optimize(
+template<typename LagrangianFunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value, bool>::type
+AugLagrangian::Optimize(
     AugLagrangianFunction<LagrangianFunctionType>& augfunc,
-    arma::mat& coordinates)
+    MatType& coordinatesIn,
+    CallbackTypes&&... callbacks)
 {
-  traits::CheckConstrainedFunctionTypeAPI<LagrangianFunctionType>();
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
+  BaseMatType& coordinates = (BaseMatType&) coordinatesIn;
+
+  // Check that the types satisfy our needs.
+  traits::CheckConstrainedFunctionTypeAPI<LagrangianFunctionType, BaseMatType,
+      BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
 
   LagrangianFunctionType& function = augfunc.Function();
 
   // Ensure that we update lambda immediately.
-  double penaltyThreshold = DBL_MAX;
+  ElemType penaltyThreshold = std::numeric_limits<ElemType>::max();
 
   // Track the last objective to compare for convergence.
-  double lastObjective = function.Evaluate(coordinates);
+  ElemType lastObjective = function.Evaluate(coordinates);
 
   // Then, calculate the current penalty.
-  double penalty = 0;
+  ElemType penalty = 0;
   for (size_t i = 0; i < function.NumConstraints(); i++)
-    penalty += std::pow(function.EvaluateConstraint(i, coordinates), 2);
+  {
+    const ElemType p = std::pow(function.EvaluateConstraint(i, coordinates), 2);
+    Callback::EvaluateConstraint(*this, function, coordinates, i, p,
+          callbacks...);
+
+    penalty += p;
+  }
 
   Info << "Penalty is " << penalty << " (threshold " << penaltyThreshold
       << ")." << std::endl;
@@ -89,36 +124,50 @@ bool AugLagrangian::Optimize(
   // The odd comparison allows user to pass maxIterations = 0 (i.e. no limit on
   // number of iterations).
   size_t it;
-  for (it = 0; it != (maxIterations - 1); it++)
+  terminate |= Callback::BeginOptimization(*this, function, coordinates,
+      callbacks...);
+  for (it = 0; it != (maxIterations - 1) && !terminate; it++)
   {
     Info << "AugLagrangian on iteration " << it
         << ", starting with objective "  << lastObjective << "." << std::endl;
 
-    if (!lbfgs.Optimize(augfunc, coordinates))
+    if (!lbfgs.Optimize(augfunc, coordinates, callbacks...))
       Info << "L-BFGS reported an error during optimization."
           << std::endl;
 
+    const ElemType objective = function.Evaluate(coordinates);
+
+    Callback::Evaluate(*this, function, coordinates, objective,
+        callbacks...);
+
     // Check if we are done with the entire optimization (the threshold we are
     // comparing with is arbitrary).
-    if (std::abs(lastObjective - function.Evaluate(coordinates)) < 1e-10 &&
+    if (std::abs(lastObjective - objective) < 1e-10 &&
         augfunc.Sigma() > 500000)
     {
       lambda = std::move(augfunc.Lambda());
       sigma = augfunc.Sigma();
+
+      Callback::EndOptimization(*this, function, coordinates, callbacks...);
       return true;
     }
 
-    lastObjective = function.Evaluate(coordinates);
+    lastObjective = objective;
 
     // Assuming that the optimization has converged to a new set of coordinates,
     // we now update either lambda or sigma.  We update sigma if the penalty
     // term is too high, and we update lambda otherwise.
 
     // First, calculate the current penalty.
-    double penalty = 0;
+    ElemType penalty = 0;
     for (size_t i = 0; i < function.NumConstraints(); i++)
     {
-      penalty += std::pow(function.EvaluateConstraint(i, coordinates), 2);
+      const ElemType p = std::pow(function.EvaluateConstraint(i, coordinates),
+          2);
+      Callback::EvaluateConstraint(*this, function, coordinates, i, p,
+          callbacks...);
+
+      penalty += p;
     }
 
     Info << "Penalty is " << penalty << " (threshold "
@@ -129,8 +178,13 @@ bool AugLagrangian::Optimize(
       // We use the update: lambda_{k + 1} = lambda_k - sigma * c(coordinates),
       // but we have to write a loop to do this for each constraint.
       for (size_t i = 0; i < function.NumConstraints(); i++)
-        augfunc.Lambda()[i] -= augfunc.Sigma() *
-            function.EvaluateConstraint(i, coordinates);
+      {
+        const ElemType p = function.EvaluateConstraint(i, coordinates);
+        Callback::EvaluateConstraint(*this, function, coordinates, i, p,
+          callbacks...);
+
+        augfunc.Lambda()[i] -= augfunc.Sigma() * p;
+      }
 
       // We also update the penalty threshold to be a factor of the current
       // penalty.
@@ -143,8 +197,12 @@ bool AugLagrangian::Optimize(
       augfunc.Sigma() *= sigmaUpdateFactor;
       Info << "Updated sigma to " << augfunc.Sigma() << "." << std::endl;
     }
+
+    terminate |= Callback::StepTaken(*this, function, coordinates,
+        callbacks...);
   }
 
+  Callback::EndOptimization(*this, function, coordinates, callbacks...);
   return false;
 }
 

include/ensmallen_bits/bigbatch_sgd/adaptive_stepsize.hpp

@@ -48,69 +48,12 @@ class AdaptiveStepsize
    * @param searchParameter The backtracking search parameter for each
    *        iteration.
    */
-  AdaptiveStepsize(const double backtrackStepSize = 0.1,
+  AdaptiveStepsize(const double backtrackStepSize = 0.5,
                    const double searchParameter = 0.1) :
       backtrackStepSize(backtrackStepSize),
       searchParameter(searchParameter)
   { /* Nothing to do here. */ }
 
-  /**
-   * This function is called in each iteration.
-   *
-   * @tparam DecomposableFunctionType Type of the function to be optimized.
-   * @param function Function to be optimized (minimized).
-   * @param stepSize Step size to be used for the given iteration.
-   * @param iterate Parameters that minimize the function.
-   * @param gradient The gradient matrix.
-   * @param gradientNorm The gradient norm to be used for the given iteration.
-   * @param offset The batch offset to be used for the given iteration.
-   * @param batchSize Batch size to be used for the given iteration.
-   * @param backtrackingBatchSize Backtracking batch size to be used for the
-   *        given iteration.
-   * @param reset Reset the step size decay parameter.
-   */
-  template<typename DecomposableFunctionType>
-  void Update(DecomposableFunctionType& function,
-              double& stepSize,
-              arma::mat& iterate,
-              const arma::mat& gradient,
-              const double gradientNorm,
-              const double sampleVariance,
-              const size_t offset,
-              const size_t batchSize,
-              const size_t backtrackingBatchSize,
-              const bool /* reset */)
-  {
-    Backtracking(function, stepSize, iterate, gradient, gradientNorm, offset,
-        backtrackingBatchSize);
-
-    // Update the iterate.
-    iterate -= stepSize * gradient;
-
-    // TODO: Develop an absolute strategy to deal with stepSizeDecay updates in
-    // case we arrive at local minima. See #1469 for more details.
-    double stepSizeDecay = 0;
-    if (gradientNorm && sampleVariance && batchSize)
-    {
-      if (batchSize < function.NumFunctions())
-      {
-        stepSizeDecay = (1 - (1 / ((double) batchSize - 1) * sampleVariance) /
-            (batchSize * gradientNorm)) / batchSize;
-      }
-      else
-      {
-        stepSizeDecay = 1 / function.NumFunctions();
-      }
-    }
-
-    // Stepsize smoothing.
-    stepSize *= (1 - ((double) batchSize / function.NumFunctions()));
-    stepSize += stepSizeDecay * ((double) batchSize / function.NumFunctions());
-
-    Backtracking(function, stepSize, iterate, gradient, gradientNorm, offset,
-        backtrackingBatchSize);
-  }
-
   //! Get the backtracking step size.
   double BacktrackStepSize() const { return backtrackStepSize; }
   //! Modify the backtracking step size.
@@ -121,48 +64,177 @@ class AdaptiveStepsize
   //! Modify the search parameter.
   double& SearchParameter() { return searchParameter; }
 
- private:
-  /**
-   * Definition of the backtracking line search algorithm based on the
-   * Armijo–Goldstein condition to determine the maximum amount to move along
-   * the given search direction.
-   *
-   * @tparam DecomposableFunctionType Type of the function to be optimized.
-   * @param function Function to be optimized (minimized).
-   * @param stepSize Step size to be used for the given iteration.
-   * @param iterate Parameters that minimize the function.
-   * @param gradient The gradient matrix.
-   * @param gradientNorm The gradient norm to be used for the given iteration.
-   * @param offset The batch offset to be used for the given iteration.
-   * @param backtrackingBatchSize The backtracking batch size.
-   */
-  template<typename DecomposableFunctionType>
-  void Backtracking(DecomposableFunctionType& function,
-                    double& stepSize,
-                    const arma::mat& iterate,
-                    const arma::mat& gradient,
-                    const double gradientNorm,
-                    const size_t offset,
-                    const size_t backtrackingBatchSize)
+
+  template<typename MatType>
+  class Policy
   {
-    double overallObjective = function.Evaluate(iterate, offset,
-        backtrackingBatchSize);
+   public:
+    // Create the instantiated object.
+    Policy(AdaptiveStepsize& parent) : parent(parent) { }
 
-    arma::mat iterateUpdate = iterate - (stepSize * gradient);
-    double overallObjectiveUpdate = function.Evaluate(iterateUpdate, offset,
-        backtrackingBatchSize);
-
-    while (overallObjectiveUpdate >
-        (overallObjective + searchParameter * stepSize * gradientNorm))
+    /**
+     * This function is called in each iteration.
+     *
+     * @tparam DecomposableFunctionType Type of the function to be optimized.
+     * @param function Function to be optimized (minimized).
+     * @param stepSize Step size to be used for the given iteration.
+     * @param iterate Parameters that minimize the function.
+     * @param gradient The gradient matrix.
+     * @param gradientNorm The gradient norm to be used for the given iteration.
+     * @param offset The batch offset to be used for the given iteration.
+     * @param batchSize Batch size to be used for the given iteration.
+     * @param backtrackingBatchSize Backtracking batch size to be used for the
+     *        given iteration.
+     * @param reset Reset the step size decay parameter.
+     */
+    template<typename DecomposableFunctionType,
+             typename GradType>
+    void Update(DecomposableFunctionType& function,
+                double& stepSize,
+                MatType& iterate,
+                GradType& gradient,
+                double& gradientNorm,
+                double& sampleVariance,
+                const size_t offset,
+                const size_t batchSize,
+                const size_t backtrackingBatchSize,
+                const bool /* reset */)
     {
-      stepSize *= backtrackStepSize;
+      Backtracking(function, stepSize, iterate, gradient, gradientNorm, offset,
+          backtrackingBatchSize);
 
-      iterateUpdate = iterate - (stepSize * gradient);
-      overallObjectiveUpdate = function.Evaluate(iterateUpdate, offset,
+      // Update the iterate.
+      iterate -= stepSize * gradient;
+
+      // Update Gradient & calculate curvature of quadratic approximation.
+      GradType functionGradient(iterate.n_rows, iterate.n_cols);
+      GradType gradPrevIterate(iterate.n_rows, iterate.n_cols);
+      GradType functionGradientPrev(iterate.n_rows, iterate.n_cols);
+
+      double vB = 0;
+      GradType delta0, delta1;
+
+      // Initialize previous iterate, if not already initialized.
+      if (iteratePrev.is_empty())
+      {
+        iteratePrev.zeros(iterate.n_rows, iterate.n_cols);
+      }
+
+      // Compute the stochastic gradient estimation.
+      function.Gradient(iterate, offset, gradient, 1);
+      function.Gradient(iteratePrev, offset, gradPrevIterate, 1);
+
+      delta1 = gradient;
+
+      for (size_t j = 1, k = 1; j < backtrackingBatchSize; ++j, ++k)
+      {
+        function.Gradient(iterate, offset + j, functionGradient, 1);
+        delta0 = delta1 + (functionGradient - delta1) / k;
+
+        // Compute sample variance.
+        vB += arma::norm(functionGradient - delta1, 2.0) *
+            arma::norm(functionGradient - delta0, 2.0);
+
+        delta1 = delta0;
+        gradient += functionGradient;
+
+        // Used for curvature calculation.
+        function.Gradient(iteratePrev, offset + j, functionGradientPrev, 1);
+        gradPrevIterate += functionGradientPrev;
+      }
+
+      // Update sample variance & norm of the gradient.
+      sampleVariance = vB;
+      gradientNorm = std::pow(arma::norm(gradient / backtrackingBatchSize, 2),
+          2.0);
+
+      // Compute curvature.
+      double v = arma::trace(arma::trans(iterate - iteratePrev) *
+          (gradient - gradPrevIterate)) /
+          std::pow(arma::norm(iterate - iteratePrev, 2), 2.0);
+
+      // Update previous iterate.
+      iteratePrev = iterate;
+
+      // TODO: Develop an absolute strategy to deal with stepSizeDecay updates
+      // in case we arrive at local minima. See #1469 for more details.
+      double stepSizeDecay = 0;
+      if (gradientNorm && sampleVariance && batchSize)
+      {
+        if (batchSize < function.NumFunctions())
+        {
+          stepSizeDecay = (1 - (1 / ((double) batchSize - 1) * sampleVariance) /
+              (batchSize * gradientNorm)) / v;
+        }
+        else
+        {
+          stepSizeDecay = 1 / v;
+        }
+      }
+
+      // Stepsize smoothing.
+      stepSize *= (1 - ((double) batchSize / function.NumFunctions()));
+      stepSize += stepSizeDecay * ((double) batchSize /
+          function.NumFunctions());
+
+      Backtracking(function, stepSize, iterate, gradient, gradientNorm, offset,
           backtrackingBatchSize);
     }
-  }
 
+    /**
+     * Definition of the backtracking line search algorithm based on the
+     * Armijo–Goldstein condition to determine the maximum amount to move along
+     * the given search direction.
+     *
+     * @tparam DecomposableFunctionType Type of the function to be optimized.
+     * @param function Function to be optimized (minimized).
+     * @param stepSize Step size to be used for the given iteration.
+     * @param iterate Parameters that minimize the function.
+     * @param gradient The gradient matrix.
+     * @param gradientNorm The gradient norm to be used for the given iteration.
+     * @param offset The batch offset to be used for the given iteration.
+     * @param backtrackingBatchSize The backtracking batch size.
+     */
+    template<typename DecomposableFunctionType,
+             typename GradType>
+    void Backtracking(DecomposableFunctionType& function,
+                      double& stepSize,
+                      const MatType& iterate,
+                      const GradType& gradient,
+                      const double gradientNorm,
+                      const size_t offset,
+                      const size_t backtrackingBatchSize)
+    {
+      typedef typename MatType::elem_type ElemType;
+
+      ElemType overallObjective = function.Evaluate(iterate,
+          offset, backtrackingBatchSize);
+
+      MatType iterateUpdate = iterate - (stepSize * gradient);
+      ElemType overallObjectiveUpdate = function.Evaluate(iterateUpdate, offset,
+          backtrackingBatchSize);
+
+      while (overallObjectiveUpdate >
+          (overallObjective - parent.searchParameter * stepSize *
+           gradientNorm))
+      {
+        stepSize *= parent.backtrackStepSize;
+
+        iterateUpdate = iterate - (stepSize * gradient);
+        overallObjectiveUpdate = function.Evaluate(iterateUpdate, offset,
+            backtrackingBatchSize);
+      }
+    }
+
+   private:
+    //! Reference to parent.
+    AdaptiveStepsize& parent;
+
+    //! Last function parameters value.
+    MatType iteratePrev;
+  };
+
+ private:
   //! The backtracking step size for each iteration.
   double backtrackStepSize;
 

include/ensmallen_bits/bigbatch_sgd/backtracking_line_search.hpp

@@ -51,53 +51,74 @@ class BacktrackingLineSearch
       searchParameter(searchParameter)
   { /* Nothing to do here. */ }
 
-  /**
-   * This function is called in each iteration.
-   *
-   * @tparam DecomposableFunctionType Type of the function to be optimized.
-   * @param function Function to be optimized (minimized).
-   * @param stepSize Step size to be used for the given iteration.
-   * @param iterate Parameters that minimize the function.
-   * @param gradient The gradient matrix.
-   * @param gradientNorm The gradient norm to be used for the given iteration.
-   * @param offset The batch offset to be used for the given iteration.
-   * @param batchSize Batch size to be used for the given iteration.
-   * @param backtrackingBatchSize Backtracking batch size to be used for the
-   *        given iteration.
-   * @param reset Reset the step size decay parameter.
-   */
-  template<typename DecomposableFunctionType>
-  void Update(DecomposableFunctionType& function,
-              double& stepSize,
-              arma::mat& iterate,
-              const arma::mat& gradient,
-              const double gradientNorm,
-              const double /* sampleVariance */,
-              const size_t offset,
-              const size_t /* batchSize */,
-              const size_t backtrackingBatchSize,
-              const bool reset)
+  //! Get the search parameter.
+  double SearchParameter() const { return searchParameter; }
+  //! Modify the search parameter.
+  double& SearchParameter() { return searchParameter; }
+
+  template<typename MatType>
+  class Policy
   {
-    if (reset)
-      stepSize *= 2;
+   public:
+    // Instantiate the policy with the given parent.
+    Policy(BacktrackingLineSearch& parent) : parent(parent) { }
 
-    double overallObjective = function.Evaluate(iterate, offset,
-        backtrackingBatchSize);
-
-    arma::mat iterateUpdate = iterate - (stepSize * gradient);
-    double overallObjectiveUpdate = function.Evaluate(iterateUpdate,
-        offset, backtrackingBatchSize);
-
-    while (overallObjectiveUpdate >
-        (overallObjective + searchParameter * stepSize * gradientNorm))
+    /**
+     * This function is called in each iteration.
+     *
+     * @tparam DecomposableFunctionType Type of the function to be optimized.
+     * @param function Function to be optimized (minimized).
+     * @param stepSize Step size to be used for the given iteration.
+     * @param iterate Parameters that minimize the function.
+     * @param gradient The gradient matrix.
+     * @param gradientNorm The gradient norm to be used for the given iteration.
+     * @param offset The batch offset to be used for the given iteration.
+     * @param batchSize Batch size to be used for the given iteration.
+     * @param backtrackingBatchSize Backtracking batch size to be used for the
+     *        given iteration.
+     * @param reset Reset the step size decay parameter.
+     */
+    template<typename DecomposableFunctionType,
+             typename GradType>
+    void Update(DecomposableFunctionType& function,
+                double& stepSize,
+                MatType& iterate,
+                GradType& gradient,
+                double& gradientNorm,
+                double& /* sampleVariance */,
+                const size_t offset,
+                const size_t /* batchSize */,
+                const size_t backtrackingBatchSize,
+                const bool reset)
     {
-      stepSize /= 2;
+      if (reset)
+        stepSize *= 2;
 
-      iterateUpdate = iterate - (stepSize * gradient);
-      overallObjectiveUpdate = function.Evaluate(iterateUpdate,
-        offset, backtrackingBatchSize);
+      typedef typename MatType::elem_type ElemType;
+
+      ElemType overallObjective = function.Evaluate(iterate, offset,
+          backtrackingBatchSize);
+
+      MatType iterateUpdate = iterate - (stepSize * gradient);
+      ElemType overallObjectiveUpdate = function.Evaluate(iterateUpdate, offset,
+          backtrackingBatchSize);
+
+      while (overallObjectiveUpdate >
+          (overallObjective - parent.searchParameter * stepSize *
+           gradientNorm))
+      {
+        stepSize /= 2;
+
+        iterateUpdate = iterate - (stepSize * gradient);
+        overallObjectiveUpdate = function.Evaluate(iterateUpdate,
+          offset, backtrackingBatchSize);
+      }
     }
-  }
+
+   private:
+    //! Reference to instantiated parent object.
+    BacktrackingLineSearch& parent;
+  };
 
  private:
   //! The search parameter for each iteration.

include/ensmallen_bits/bigbatch_sgd/bigbatch_sgd.hpp

@@ -58,7 +58,7 @@ namespace ens {
  *   title   = {Big Batch {SGD:} Automated Inference using Adaptive Batch
  *              Sizes},
  *   author  = {Soham De and Abhay Kumar Yadav and David W. Jacobs and
-                Tom Goldstein},
+ *              Tom Goldstein},
  *   journal = {CoRR},
  *   year    = {2017},
  *   url     = {http://arxiv.org/abs/1610.05792},
@@ -91,26 +91,52 @@ class BigBatchSGD
    * @param tolerance Maximum absolute tolerance to terminate algorithm.
    * @param shuffle If true, the batch order is shuffled; otherwise, each
    *        batch is visited in linear order.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
    */
   BigBatchSGD(const size_t batchSize = 1000,
               const double stepSize = 0.01,
               const double batchDelta = 0.1,
               const size_t maxIterations = 100000,
               const double tolerance = 1e-5,
-              const bool shuffle = true);
+              const bool shuffle = true,
+              const bool exactObjective = false);
   /**
    * Optimize the given function using big-batch SGD.  The given starting point
    * will be modified to store the finishing point of the algorithm, and the
    * final objective value is returned.
    *
    * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function,
-                  arma::mat& iterate);
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(DecomposableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(DecomposableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<DecomposableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
 
   //! Get the batch size.
   size_t BatchSize() const { return batchSize; }
@@ -147,6 +173,11 @@ class BigBatchSGD
   //! Modify the update policy.
   UpdatePolicyType& UpdatePolicy() { return updatePolicy; }
 
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return exactObjective; }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return exactObjective; }
+
  private:
   //! The size of the current batch.
   size_t batchSize;
@@ -167,8 +198,14 @@ class BigBatchSGD
   //! iterating.
   bool shuffle;
 
+  //! Controls whether or not the actual Objective value is calculated.
+  bool exactObjective;
+
   //! The update policy used to update the parameters in each iteration.
   UpdatePolicyType updatePolicy;
+
+  //! Instantiated update policy.
+  Any instUpdatePolicy;
 };
 
 using BBS_Armijo = BigBatchSGD<BacktrackingLineSearch>;

include/ensmallen_bits/bigbatch_sgd/bigbatch_sgd_impl.hpp

@@ -26,76 +26,82 @@ BigBatchSGD<UpdatePolicyType>::BigBatchSGD(
     const double batchDelta,
     const size_t maxIterations,
     const double tolerance,
-    const bool shuffle) :
+    const bool shuffle,
+    const bool exactObjective) :
     batchSize(batchSize),
     stepSize(stepSize),
     batchDelta(batchDelta),
     maxIterations(maxIterations),
     tolerance(tolerance),
     shuffle(shuffle),
+    exactObjective(exactObjective),
     updatePolicy(UpdatePolicyType())
 { /* Nothing to do. */ }
 
 //! Optimize the function (minimize).
 template<typename UpdatePolicyType>
-template<typename DecomposableFunctionType>
-double BigBatchSGD<UpdatePolicyType>::Optimize(
-    DecomposableFunctionType& function, arma::mat& iterate)
+template<typename DecomposableFunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value,
+typename MatType::elem_type>::type
+BigBatchSGD<UpdatePolicyType>::Optimize(
+    DecomposableFunctionType& function,
+    MatType& iterateIn,
+    CallbackTypes&&... callbacks)
 {
-  typedef Function<DecomposableFunctionType> FullFunctionType;
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
+  typedef Function<DecomposableFunctionType, BaseMatType, BaseGradType>
+      FullFunctionType;
   FullFunctionType& f(static_cast<FullFunctionType&>(function));
 
   // Make sure we have all the methods that we need.
-  traits::CheckDecomposableFunctionTypeAPI<FullFunctionType>();
+  traits::CheckDecomposableFunctionTypeAPI<FullFunctionType, BaseMatType,
+      BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  typedef typename UpdatePolicyType::template Policy<BaseMatType>
+      InstUpdatePolicyType;
+
+  if (!instUpdatePolicy.Has<InstUpdatePolicyType>())
+  {
+    instUpdatePolicy.Clean();
+    instUpdatePolicy.Set<InstUpdatePolicyType>(
+        new InstUpdatePolicyType(updatePolicy));
+  }
 
   // Find the number of functions to use.
   const size_t numFunctions = f.NumFunctions();
 
   // To keep track of where we are and how things are going.
   size_t currentFunction = 0;
-  double overallObjective = 0;
-  double lastObjective = DBL_MAX;
+  size_t epoch = 1;
+  ElemType overallObjective = 0;
+  ElemType lastObjective = DBL_MAX;
   bool reset = false;
-  arma::mat delta0, delta1;
+  BaseGradType delta0, delta1;
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
 
   // Now iterate!
-  arma::mat gradient(iterate.n_rows, iterate.n_cols);
-  arma::mat functionGradient(iterate.n_rows, iterate.n_cols);
+  BaseGradType gradient(iterate.n_rows, iterate.n_cols);
+  BaseGradType functionGradient(iterate.n_rows, iterate.n_cols);
   const size_t actualMaxIterations = (maxIterations == 0) ?
       std::numeric_limits<size_t>::max() : maxIterations;
-  for (size_t i = 0; i < actualMaxIterations; /* incrementing done manually */)
+  terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  for (size_t i = 0; i < actualMaxIterations && !terminate;
+      /* incrementing done manually */)
   {
-    // Is this iteration the start of a sequence?
-    if ((currentFunction % numFunctions) == 0 && i > 0)
-    {
-      // Output current objective function.
-      Info << "Big-batch SGD: iteration " << i << ", objective "
-          << overallObjective << "." << std::endl;
-
-      if (std::isnan(overallObjective) || std::isinf(overallObjective))
-      {
-        Warn << "Big-batch SGD: converged to " << overallObjective
-            << "; terminating with failure.  Try a smaller step size?"
-            << std::endl;
-        return overallObjective;
-      }
-
-      if (std::abs(lastObjective - overallObjective) < tolerance)
-      {
-        Info << "Big-batch SGD: minimized within tolerance " << tolerance
-            << "; terminating optimization." << std::endl;
-        return overallObjective;
-      }
-
-      // Reset the counter variables.
-      lastObjective = overallObjective;
-      overallObjective = 0;
-      currentFunction = 0;
-
-      if (shuffle) // Determine order of visitation.
-        f.Shuffle();
-    }
-
     // Find the effective batch size; we have to take the minimum of three
     // things:
     // - the batch size can't be larger than the user-specified batch size;
@@ -112,10 +118,16 @@ double BigBatchSGD<UpdatePolicyType>::Optimize(
     // Compute the stochastic gradient estimation.
     f.Gradient(iterate, currentFunction, gradient, 1);
 
+    terminate |= Callback::Gradient(*this, f, iterate, gradient, callbacks...);
+
     delta1 = gradient;
     for (size_t j = 1; j < effectiveBatchSize; ++j, ++k)
     {
       f.Gradient(iterate, currentFunction + j, functionGradient, 1);
+
+      terminate |= Callback::Gradient(*this, f, iterate, functionGradient,
+          callbacks...);
+
       delta0 = delta1 + (functionGradient - delta1) / k;
 
       // Compute sample variance.
@@ -150,6 +162,9 @@ double BigBatchSGD<UpdatePolicyType>::Optimize(
         for (size_t j = 0; j < batchOffset; ++j, ++k)
         {
           f.Gradient(iterate, batchStart + j, functionGradient, 1);
+          terminate |= Callback::Gradient(*this, f, iterate,
+              functionGradient, callbacks...);
+
           delta0 = delta1 + (functionGradient - delta1) / (k + 1);
 
           // Compute sample variance.
@@ -170,29 +185,83 @@ double BigBatchSGD<UpdatePolicyType>::Optimize(
       }
     }
 
-    updatePolicy.Update(f, stepSize, iterate, gradient, gB, vB,
-        currentFunction, batchSize, effectiveBatchSize, reset);
+    instUpdatePolicy.As<InstUpdatePolicyType>().Update(f, stepSize, iterate,
+        gradient, gB, vB, currentFunction, batchSize, effectiveBatchSize,
+        reset);
 
     // Update the iterate.
     iterate -= stepSize * gradient;
+    terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
 
-    overallObjective += f.Evaluate(iterate, currentFunction,
+    const ElemType objective = f.Evaluate(iterate, currentFunction,
         effectiveBatchSize);
+    overallObjective += objective;
+
+    terminate |= Callback::Evaluate(*this, f, iterate, objective,
+        callbacks...);
 
     i += effectiveBatchSize;
     currentFunction += effectiveBatchSize;
+
+    // Is this iteration the start of a sequence?
+    if ((currentFunction % numFunctions) == 0)
+    {
+      terminate |= Callback::EndEpoch(*this, f, iterate, epoch++,
+          overallObjective / (ElemType) numFunctions, callbacks...);
+
+      // Output current objective function.
+      Info << "Big-batch SGD: iteration " << i << ", objective "
+          << overallObjective << "." << std::endl;
+
+      if (std::isnan(overallObjective) || std::isinf(overallObjective))
+      {
+        Warn << "Big-batch SGD: converged to " << overallObjective
+            << "; terminating with failure.  Try a smaller step size?"
+            << std::endl;
+
+        Callback::EndOptimization(*this, f, iterate, callbacks...);
+        return overallObjective;
+      }
+
+      if (std::abs(lastObjective - overallObjective) < tolerance ||
+          Callback::BeginEpoch(*this, f, iterate, epoch, overallObjective,
+              callbacks...))
+      {
+        Info << "Big-batch SGD: minimized within tolerance " << tolerance
+            << "; terminating optimization." << std::endl;
+
+        Callback::EndOptimization(*this, f, iterate, callbacks...);
+        return overallObjective;
+      }
+
+      // Reset the counter variables.
+      lastObjective = overallObjective;
+      overallObjective = 0;
+      currentFunction = 0;
+
+      if (shuffle) // Determine order of visitation.
+        f.Shuffle();
+    }
   }
 
   Info << "Big-batch SGD: maximum iterations (" << maxIterations << ") "
       << "reached; terminating optimization." << std::endl;
 
-  // Calculate final objective.
-  overallObjective = 0;
-  for (size_t i = 0; i < numFunctions; i += batchSize)
+  // Calculate final objective if exactObjective is set to true.
+  if (exactObjective)
   {
-    const size_t effectiveBatchSize = std::min(batchSize, numFunctions - i);
-    overallObjective += f.Evaluate(iterate, i, effectiveBatchSize);
+    overallObjective = 0;
+    for (size_t i = 0; i < numFunctions; i += batchSize)
+    {
+      const size_t effectiveBatchSize = std::min(batchSize, numFunctions - i);
+      const ElemType objective = f.Evaluate(iterate, i, effectiveBatchSize);
+      overallObjective += objective;
+
+      Callback::Evaluate(*this, f, iterate, objective, callbacks...);
+    }
   }
+
+  Callback::EndOptimization(*this, f, iterate, callbacks...);
   return overallObjective;
 }
 

include/ensmallen_bits/callbacks/callbacks.hpp

@@ -0,0 +1,767 @@
+/**
+ * @file callbacks.hpp
+ * @author Marcus Edel
+ *
+ * The Callback class will invoke the specified callbacks.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_CALLBACKS_HPP
+#define ENSMALLEN_CALLBACKS_CALLBACKS_HPP
+
+#include <ensmallen_bits/callbacks/traits.hpp>
+
+namespace ens {
+
+/**
+ * Callbacks are a set of functions that can be applied at given stages of the
+ * optimization process. The following callbacks are available:
+ *
+ * - Evaluate(optimizer, function, coordinates, objective):
+ *   called after any call to Evaluate().
+ *
+ * - StepTaken(optimizer, function, coordinates):
+ *   called after any step is taken.
+ *
+ * - Gradient(optimizer, function, coordinates, gradient):
+ *   called whenever the gradient is computed.
+ *
+ * - BeginEpoch(optimizer, function, coordinates, epoch, objective):
+ *   called at the beginning of a pass over the data. The objective may be
+ *   exact or an estimate depending on exactObjective's value.
+ *
+ * - EvaluateConstraint(optimizer, function, coordinates, constraint,
+ *                      constraintValue):
+ *   called after any call to EvaluateConstraint().
+ *
+ * - GradientConstraint(optimizer, function, coordinates, constraint,
+ *                      constraintGradient):
+ *   called after any call to GradientConstraint().
+ *
+ * - BeginOptimization(optimizer, function, coordinates):
+ *   called at the beginning of the optimization.
+ *
+ * - EndOptimization(optimizer, function, coordinates):
+ *   called at the end of the optimization.
+ */
+class Callback
+{
+ public:
+  /**
+   * Invoke the BeginOptimization() callback if it exists.
+   *
+   * @param callback The callback to call.
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<
+      callbacks::traits::HasBeginOptimizationSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasBool,
+      bool>::type
+  BeginOptimizationFunction(CallbackType& callback,
+                            OptimizerType& optimizer,
+                            FunctionType& function,
+                            MatType& coordinates)
+  {
+    return const_cast<CallbackType&>(callback).BeginOptimization(optimizer,
+        function, coordinates);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<
+      callbacks::traits::HasBeginOptimizationSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasVoid,
+      bool>::type
+  BeginOptimizationFunction(CallbackType& callback,
+                            OptimizerType& optimizer,
+                            FunctionType& function,
+                            MatType& coordinates)
+  {
+    const_cast<CallbackType&>(callback).BeginOptimization(optimizer, function,
+        coordinates);
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<
+      callbacks::traits::HasBeginOptimizationSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasNone,
+      bool>::type
+  BeginOptimizationFunction(CallbackType& /* callback */,
+                            OptimizerType& /* optimizer */,
+                            FunctionType& /* function */,
+                            MatType& /* coordinates */)
+  { return false; }
+
+  /**
+   * Iterate over the callbacks and invoke the BeginOptimization() callback if
+   * it exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  static bool BeginOptimization(OptimizerType& optimizer,
+                                FunctionType& function,
+                                MatType& coordinates,
+                                CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::BeginOptimizationFunction(callbacks, optimizer,
+            function, coordinates)... };
+     return result;
+  }
+
+  /**
+   * Invoke the EndOptimization() callback if it exists.
+   *
+   * @param callback The callback to call.
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<callbacks::traits::HasEndOptimizationSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      bool>::type
+  EndOptimizationFunction(CallbackType& callback,
+                          OptimizerType& optimizer,
+                          FunctionType& function,
+                          MatType& coordinates)
+  {
+    return (const_cast<CallbackType&>(callback).EndOptimization(
+        optimizer, function, coordinates), false);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<!callbacks::traits::HasEndOptimizationSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      bool>::type
+  EndOptimizationFunction(CallbackType& /* callback */,
+                          OptimizerType& /* optimizer */,
+                          FunctionType& /* function */,
+                          MatType& /* coordinates */)
+  { return false; }
+
+  /**
+   * Iterate over the callbacks and invoke the EndOptimization() callback if it
+   * exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  static bool EndOptimization(OptimizerType& optimizer,
+                              FunctionType& function,
+                              MatType& coordinates,
+                              CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::EndOptimizationFunction(callbacks, optimizer,
+            function, coordinates)... };
+     return result;
+  }
+
+  /**
+   * Invoke the Evaluate() callback if it exists.
+   *
+   * @param callback The callback to call.
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param objective Objective value of the current point.
+   */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<callbacks::traits::HasEvaluateSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      bool>::type
+  EvaluateFunction(CallbackType& callback,
+                   OptimizerType& optimizer,
+                   FunctionType& function,
+                   const MatType& coordinates,
+                   const double objective)
+  {
+    return (const_cast<CallbackType&>(callback).Evaluate(
+        optimizer, function, coordinates, objective), false);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<!callbacks::traits::HasEvaluateSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      bool>::type
+  EvaluateFunction(CallbackType& /* callback */,
+                   OptimizerType& /* optimizer */,
+                   FunctionType& /* function */,
+                   const MatType& /* coordinates */,
+                   const double /* objective */)
+  { return false; }
+
+  /**
+   * Iterate over the callbacks and invoke the Evaluate() callback if it exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param objective Objective value of the current point.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  static bool Evaluate(OptimizerType& optimizer,
+                       FunctionType& function,
+                       const MatType& coordinates,
+                       const double objective,
+                       CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::EvaluateFunction(callbacks, optimizer, function,
+        coordinates, objective)... };
+     return result;
+  }
+
+  /**
+   * Invoke the EvaluateConstraint() callback if it exists.
+   *
+   * @param callback The callback to call.
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param constraint The index of the constraint.
+   * @param constraintValue Constraint value of the current point.
+   */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<
+      callbacks::traits::HasEvaluateConstraintSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      bool>::type
+  EvaluateConstraintFunction(CallbackType& callback,
+                             OptimizerType& optimizer,
+                             FunctionType& function,
+                             const MatType& coordinates,
+                             const size_t constraint,
+                             const double constraintValue)
+  {
+    return (const_cast<CallbackType&>(callback).EvaluateConstraint(
+        optimizer, function, coordinates, constraint, constraintValue), false);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<
+      !callbacks::traits::HasEvaluateConstraintSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      bool>::type
+  EvaluateConstraintFunction(CallbackType& /* callback */,
+                             OptimizerType& /* optimizer */,
+                             FunctionType& /* function */,
+                             const MatType& /* coordinates */,
+                             const size_t /* constraint */,
+                             const double /* constraintValue */)
+  { return false; }
+
+  /**
+   * Iterate over the callbacks and invoke the EvaluateConstraint() callback if
+   * it exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param constraint The index of the constraint.
+   * @param constraintValue Constraint value of the current point.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  static bool EvaluateConstraint(OptimizerType& optimizer,
+                                 FunctionType& function,
+                                 const MatType& coordinates,
+                                 const size_t constraint,
+                                 const double constraintValue,
+                                 CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::EvaluateConstraintFunction(callbacks, optimizer,
+            function, coordinates, constraint, constraintValue)... };
+     return result;
+  }
+
+  /**
+   * Invoke the Gradient() callback if it exists.
+   *
+   * @param callback The callback to call.
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param gradient Matrix that holds the gradient.
+   */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType>
+  static typename std::enable_if<callbacks::traits::HasGradientSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::value,
+      bool>::type
+  GradientFunction(CallbackType& callback,
+                   OptimizerType& optimizer,
+                   FunctionType& function,
+                   const MatType& coordinates,
+                   GradType& gradient)
+  {
+    return (const_cast<CallbackType&>(callback).Gradient(
+        optimizer, function, coordinates, gradient), false);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType>
+  static typename std::enable_if<!callbacks::traits::HasGradientSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::value,
+      bool>::type
+  GradientFunction(CallbackType& /* callback */,
+                   OptimizerType& /* optimizer */,
+                   FunctionType& /* function */,
+                   const MatType& /* coordinates */,
+                   GradType& /* gradient */)
+  { return false; }
+
+  /**
+   * Iterate over the callbacks and invoke the Gradient() callback if it exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param gradient Matrix that holds the gradient.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  static bool Gradient(OptimizerType& optimizer,
+                       FunctionType& function,
+                       const MatType& coordinates,
+                       GradType& gradient,
+                       CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::GradientFunction(callbacks, optimizer, function,
+        coordinates, gradient)... };
+     return result;
+  }
+
+  /**
+   * Invoke the GradientConstraint() callback if it exists.
+   *
+   * @param callback The callback to call.
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param gradient Matrix that holds the gradient.
+   */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType>
+  static typename std::enable_if<
+      callbacks::traits::HasGradientConstraintSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::value,
+      bool>::type
+  GradientConstraintFunction(CallbackType& callback,
+                             OptimizerType& optimizer,
+                             FunctionType& function,
+                             const MatType& coordinates,
+                             const size_t constraint,
+                             GradType& gradient)
+  {
+    return (const_cast<CallbackType&>(callback).GradientConstraint(
+        optimizer, function, coordinates, constraint, gradient), false);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType>
+  static typename std::enable_if<
+      !callbacks::traits::HasGradientConstraintSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::value,
+      bool>::type
+  GradientConstraintFunction(CallbackType& /* callback */,
+                             OptimizerType& /* optimizer */,
+                             FunctionType& /* function */,
+                             const MatType& /* coordinates */,
+                             const size_t /* constraint */,
+                             GradType& /* gradient */)
+  { return false; }
+
+  /**
+   * Iterate over the callbacks and invoke the GradientConstraint() callback if
+   * it exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param gradient Matrix that holds the gradient.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  static bool Gradient(OptimizerType& optimizer,
+                       FunctionType& function,
+                       const MatType& coordinates,
+                       const size_t constraint,
+                       GradType& gradient,
+                       CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::GradientConstraintFunction(callbacks, optimizer,
+        function, coordinates, constraint, gradient)... };
+     return result;
+  }
+
+  /**
+   * Iterate over the callbacks and invoke the Evaluate() and Gradient()
+   * callback if it exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param objective Objective value of the current point.
+   * @param gradient Matrix that holds the gradient.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  static bool EvaluateWithGradient(OptimizerType& optimizer,
+                                   FunctionType& function,
+                                   const MatType& coordinates,
+                                   const double objective,
+                                   GradType& gradient,
+                                   CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::EvaluateFunction(callbacks, optimizer, function,
+        coordinates, objective)... };
+
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::GradientFunction(callbacks, optimizer, function,
+        coordinates, gradient)... };
+     return result;
+  }
+
+  /**
+   * Invoke the BeginEpoch() callback if it exists.
+   *
+   * @param callback The callback to call.
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<callbacks::traits::HasBeginEpochSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::value, bool>::type
+  BeginEpochFunction(CallbackType& callback,
+                     OptimizerType& optimizer,
+                     FunctionType& function,
+                     const MatType& coordinates,
+                     const size_t epoch,
+                     const double objective)
+  {
+    return (const_cast<CallbackType&>(callback).BeginEpoch(
+        optimizer, function, coordinates, epoch, objective), false);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<!callbacks::traits::HasBeginEpochSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::value, bool>::type
+  BeginEpochFunction(CallbackType& /* callback  */,
+                     OptimizerType& /* optimizer */,
+                     FunctionType& /* function  */,
+                     const MatType& /* coordinates  */,
+                     const size_t /* epoch  */,
+                     const double /* objective */)
+  { return false; }
+
+  /**
+   * Iterate over all callbacks and invoke the BeginEpoch() callback if it
+   * exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  static bool BeginEpoch(OptimizerType& optimizer,
+                         FunctionType& function,
+                         const MatType& coordinates,
+                         const size_t epoch,
+                         const double objective,
+                         CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::BeginEpochFunction(callbacks, optimizer, function,
+        coordinates, epoch, objective)... };
+     return result;
+  }
+
+  /**
+   * Invoke the EndEpoch() callback if it exists.
+   *
+   * @param callback The callback to call.
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<callbacks::traits::HasEndEpochSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasBool, bool>::type
+  EndEpochFunction(CallbackType& callback,
+                   OptimizerType& optimizer,
+                   FunctionType& function,
+                   const MatType& coordinates,
+                   const size_t epoch,
+                   const double objective)
+  {
+    return const_cast<CallbackType&>(callback).EndEpoch(
+        optimizer, function, coordinates, epoch, objective);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<callbacks::traits::HasEndEpochSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasVoid, bool>::type
+  EndEpochFunction(CallbackType& callback,
+                   OptimizerType& optimizer,
+                   FunctionType& function,
+                   const MatType& coordinates,
+                   const size_t epoch,
+                   const double objective)
+  {
+    const_cast<CallbackType&>(callback).EndEpoch(
+        optimizer, function, coordinates, epoch, objective);
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<callbacks::traits::HasEndEpochSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasNone, bool>::type
+  EndEpochFunction(CallbackType& /* callback */,
+                   OptimizerType& /* optimizer */,
+                   FunctionType& /* function */,
+                   const MatType& /* coordinates */,
+                   const size_t /* epoch */,
+                   const double /* objective */)
+  { return false; }
+
+  /**
+   * Iterate over all callbacks and invoke the EndEpoch() callback if it exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  static bool EndEpoch(OptimizerType& optimizer,
+                       FunctionType& function,
+                       const MatType& coordinates,
+                       const size_t epoch,
+                       const double objective,
+                       CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::EndEpochFunction(callbacks, optimizer, function,
+        coordinates, epoch, objective)... };
+     return result;
+  }
+
+  /**
+   * Invoke the StepTaken() callback if it exists.
+   *
+   * @param callback The callback to call.
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<
+      callbacks::traits::HasStepTakenSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasBool,
+      bool>::type
+  StepTakenFunction(CallbackType& callback,
+                    OptimizerType& optimizer,
+                    FunctionType& function,
+                    MatType& coordinates)
+  {
+    return const_cast<CallbackType&>(callback).StepTaken(optimizer,
+        function, coordinates);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<
+      callbacks::traits::HasStepTakenSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasVoid,
+      bool>::type
+  StepTakenFunction(CallbackType& callback,
+                    OptimizerType& optimizer,
+                    FunctionType& function,
+                    MatType& coordinates)
+  {
+    const_cast<CallbackType&>(callback).StepTaken(optimizer, function,
+        coordinates);
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<
+      callbacks::traits::HasStepTakenSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasNone,
+      bool>::type
+  StepTakenFunction(CallbackType& /* callback */,
+                    OptimizerType& /* optimizer */,
+                    FunctionType& /* function */,
+                    MatType& /* coordinates */)
+  { return false; }
+
+  /**
+   * Iterate over the callbacks and invoke the StepTaken() callback if it
+   * exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  static bool StepTaken(OptimizerType& optimizer,
+                        FunctionType& function,
+                        MatType& coordinates,
+                        CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::StepTakenFunction(callbacks, optimizer,
+            function, coordinates)... };
+     return result;
+  }
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/early_stop_at_min_loss.hpp

@@ -0,0 +1,83 @@
+/**
+ * @file early_stop_at_min_loss.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of the early stop at minimum loss callback function.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_EARLY_STOP_AT_MIN_LOSS_HPP
+#define ENSMALLEN_CALLBACKS_EARLY_STOP_AT_MIN_LOSS_HPP
+
+namespace ens {
+
+/**
+ * Early stopping to terminate the optimization process early if the loss stops
+ * decreasing.
+ */
+class EarlyStopAtMinLoss
+{
+ public:
+  /**
+   * Set up the early stop at min loss class, which keeps track of the minimum
+   * loss and stops the optimization process if the loss stops decreasing.
+   *
+   * @param patienceIn The number of epochs to wait after the minimum loss has
+   *    been reached or no improvement has been made (Default: 10).
+   */
+  EarlyStopAtMinLoss(const size_t patienceIn = 10) :
+      patience(patienceIn),
+      bestObjective(std::numeric_limits<double>::max()),
+      steps(0)
+  { /* Nothing to do here */ }
+
+  /**
+   * Callback function called at the end of a pass over the data.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  bool EndEpoch(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                const size_t /* epoch */,
+                const double objective)
+  {
+    if (objective < bestObjective)
+    {
+      steps = 0;
+      bestObjective = objective;
+      return false;
+    }
+
+    steps++;
+    if (steps >= patience)
+    {
+      Info << "Minimum loss reached; terminate optimization." << std::endl;
+      return true;
+    }
+
+    return false;
+  }
+
+ private:
+  //! The number of epochs to wait before terminating the optimization process.
+  size_t patience;
+
+  //! Locally-stored best objective.
+  double bestObjective;
+
+  //! Locally-stored number of steps since the loss improved.
+  size_t steps;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/print_loss.hpp

@@ -0,0 +1,57 @@
+/**
+ * @file print_loss.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of the print loss callback function.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_PRINT_LOSS_HPP
+#define ENSMALLEN_CALLBACKS_PRINT_LOSS_HPP
+
+namespace ens {
+
+/**
+ * Print loss function, based on the EndEpoch callback function.
+ */
+class PrintLoss
+{
+ public:
+  /**
+   * Set up the print loss callback class with the width and output stream.
+   *
+   * @param ostream Ostream which receives output from this object.
+   */
+  PrintLoss(std::ostream& output = arma::get_cout_stream()) : output(output)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Callback function called at the end of a pass over the data.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void EndEpoch(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                const size_t /* epoch */,
+                const double objective)
+  {
+    output << objective << std::endl;
+  }
+
+ private:
+  //! The output stream that all data is to be sent to; example: std::cout.
+  std::ostream& output;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/progress_bar.hpp

@@ -0,0 +1,265 @@
+/**
+ * @file progress_bar.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of a simple progress bar callback function.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_PROGRESS_BAR_HPP
+#define ENSMALLEN_CALLBACKS_PROGRESS_BAR_HPP
+
+namespace ens {
+
+/**
+ * A simple progress bar, based on the maximum number of optimizer iterations,
+ * batch-size, number of functions and the StepTaken callback function.
+ */
+class ProgressBar
+{
+ public:
+  /**
+   * Set up the progress bar callback class with the given width and output
+   * stream.
+   *
+   * @param widthIn Width of the bar.
+   * @param ostream Ostream which receives output from this object.
+   */
+  ProgressBar(const size_t widthIn = 70,
+              std::ostream& output = arma::get_cout_stream()) :
+      width(100.0 / widthIn),
+      output(output),
+      objective(0),
+      epochs(0),
+      epochSize(0),
+      step(1),
+      steps(0),
+      newEpoch(false),
+      epoch(1)
+
+  { /* Nothing to do here. */ }
+
+  /**
+   * Callback function called at the begin of the optimization process.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void BeginOptimization(OptimizerType& optimizer,
+                         FunctionType& function,
+                         MatType& /* coordinates */)
+  {
+    static_assert(callbacks::traits::HasBatchSizeSignature<
+      OptimizerType>::value,
+      "The OptimizerType does not have a correct definition of BatchSize(). "
+      "Please check that the OptimizerType fully satisfies the requirements "
+      "of the ProgressBar API; see the callbacks documentation for more "
+      "details.");
+
+    static_assert(callbacks::traits::HasMaxIterationsSignature<
+      OptimizerType>::value,
+      "The OptimizerType does not have a correct definition of MaxIterations()."
+      " Please check that the OptimizerType fully satisfies the requirements "
+      "of the ProgressBar API; see the callbacks documentation for more "
+      "details.");
+
+    static_assert(callbacks::traits::HasNumFunctionsSignature<
+      FunctionType>::value,
+      "The OptimizerType does not have a correct definition of NumFunctions(). "
+      "Please check that the OptimizerType fully satisfies the requirements "
+      "of the ProgressBar API; see the callbacks documentation for more "
+      "details.");
+
+    epochSize = function.NumFunctions() / optimizer.BatchSize();
+    if (function.NumFunctions() % optimizer.BatchSize() > 0)
+      epochSize++;
+
+    if (!optimizer.MaxIterations())
+    {
+      Warn << "Maximum number of iterations not defined (no limit),"
+           << " no progress bar shown." << std::endl;
+    }
+    else
+    {
+      epochs = optimizer.MaxIterations() / epochSize;
+      if (optimizer.MaxIterations() % epochSize > 0)
+        epochs++;
+    }
+
+    stepTimer.tic();
+  }
+
+  /**
+   * Callback function called at the begin of a pass over the data.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epochIn The index of the current epoch.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void BeginEpoch(OptimizerType& /* optimizer */,
+                  FunctionType& /* function */,
+                  const MatType& /* coordinates */,
+                  const size_t epochIn,
+                  const double /* objective */)
+  {
+    // Start the timer.
+    epochTimer.tic();
+
+    // Reset epoch parameter.
+    objective = 0;
+    step = 1;
+
+    epoch = epochIn;
+    newEpoch = true;
+  }
+
+  /**
+   * Callback function called once a step is taken.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void StepTaken(OptimizerType& /* optimizer */,
+                 FunctionType& /* function */,
+                 const MatType& /* coordinates */)
+  {
+    if (newEpoch)
+    {
+      output << "Epoch " << epoch << "/" << epochs << "\n";
+      output.flush();
+      newEpoch = false;
+    }
+
+    const size_t progress = ((double) step / epochSize) * 100;
+    output << step++ << "/" << epochSize << " [";
+    for (size_t i = 0; i < 100; i += width)
+    {
+      if (i < progress)
+      {
+        output << "=";
+      }
+      else if (i == progress)
+      {
+        output << ">";
+      }
+      else
+      {
+        output << ".";
+      }
+    }
+
+    output << "] " << progress << "% - ETA: " << (size_t) stepTimer.toc() *
+        (epochSize - step + 1) % 60 << "s - loss: " <<
+        objective / (double) step <<  "\r";
+    output.flush();
+
+    stepTimer.tic();
+  }
+
+  /**
+   * Callback function called at any call to Evaluate().
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param objectiveIn Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void Evaluate(OptimizerType& optimizer,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                const double objectiveIn)
+  {
+    objective += objectiveIn / optimizer.BatchSize();
+    steps++;
+  }
+
+  /**
+   * Callback function called at the end of a pass over the data.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void EndEpoch(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                const size_t /* epoch */,
+                const double objective)
+  {
+    const size_t progress = ((double) (step - 1) / epochSize) * 100;
+    output << step - 1 << "/" << epochSize << " [";
+    for (size_t i = 0; i < 100; i += width)
+    {
+      if (i < progress)
+      {
+        output << "=";
+      }
+      else if (i == progress)
+      {
+        output << ">";
+      }
+      else
+      {
+        output << ".";
+      }
+    }
+
+    const size_t stepTime = epochTimer.toc() / (double) epochSize * 1000;
+    output << "] " << progress << "% - " << (size_t) epochTimer.toc() % 60
+        << "s " << stepTime << "ms/step " << "- loss: " << objective  <<  "\n";
+    output.flush();
+  }
+
+ private:
+  //! Length of a single step (1%).
+  double width;
+
+  //! The output stream that all data is to be sent to; example: std::cout.
+  std::ostream& output;
+
+  //! Objective over the current epoch.
+  double objective;
+
+  //! Total number of epochs
+  size_t epochs;
+
+  //! Number of steps per epoch.
+  size_t epochSize;
+
+  //! Current step number.
+  size_t step;
+
+  //! Number of steps taken.
+  size_t steps;
+
+  //! Indicates a new epoch.
+  bool newEpoch;
+
+  //! Locally-stored epoch.
+  size_t epoch;
+
+  //! Locally-stored step timer object.
+  arma::wall_clock stepTimer;
+
+  //! Locally-stored epoch timer object.
+  arma::wall_clock epochTimer;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/store_best_coordinates.hpp

@@ -0,0 +1,75 @@
+/**
+ * @file store_best_coordinates.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of the store best coordinates callback function.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_STORE_BEST_COORDINATES_HPP
+#define ENSMALLEN_CALLBACKS_STORE_BEST_COORDINATES_HPP
+
+namespace ens {
+
+/**
+ * Store best coordinates function, based on the Evaluate callback function.
+ *
+ * @tparam MatType Type of the model coordinates (arma::colvec, arma::mat,
+ *     arma::sp_mat or arma::cube).
+ */
+template<typename ModelMatType = arma::mat>
+class StoreBestCoordinates
+{
+ public:
+  /**
+   * Set up the store best model class, which keeps the best-performing
+   * coordinates and objective.
+   */
+  StoreBestCoordinates() : bestObjective(std::numeric_limits<double>::max())
+  { /* Nothing to do here. */ }
+
+  /**
+   * Callback function called after any call to Evaluate().
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void Evaluate(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& coordinates,
+                const double objective)
+  {
+    if (objective < bestObjective)
+    {
+      bestObjective = objective;
+      bestCoordinates = coordinates;
+    }
+  }
+
+  //! Get the best coordinates.
+  ModelMatType const& BestCoordinates() const { return bestCoordinates; }
+  //! Modify the best coordinates.
+  ModelMatType& BestCoordinatesl() { return bestCoordinates; }
+
+  //! Get the best objective.
+  double const& BestObjective() const { return bestObjective; }
+  //! Modify the best objective.
+  double& BestObjective() { return bestObjective; }
+
+ private:
+  //! Locally-stored best objective.
+  double bestObjective;
+
+  //! Locally-stored best model coordinates.
+  ModelMatType bestCoordinates;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/timer_stop.hpp

@@ -0,0 +1,83 @@
+/**
+ * @file timer_stop.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of the timer stop callback function.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_TIMER_STOP_HPP
+#define ENSMALLEN_CALLBACKS_TIMER_STOP_HPP
+
+namespace ens {
+
+/**
+ * Timer stop function, is based on the BeginOptimization callback function to
+ * start the timer and the EndEpoch callback function to update the timer.
+ */
+class TimerStop
+{
+ public:
+  /**
+   * Set up the print loss callback class with the width and output stream.
+   *
+   * @param durationIn The duration of the timer in seconds.
+   */
+  TimerStop(const double durationIn) : duration(durationIn)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Callback function called at the start of the optimization process.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void BeginOptimization(OptimizerType& /* optimizer */,
+                         FunctionType& /* function */,
+                         MatType& /* coordinates */)
+  {
+    // Start the timer.
+    timer.tic();
+  }
+
+  /**
+   * Callback function called at the end of a pass over the data.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  bool EndEpoch(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                const size_t /* epoch */,
+                const double /* objective */)
+  {
+    if (timer.toc() > duration)
+    {
+      Info << "Timer timeout reached; terminate optimization." << std::endl;
+      return true;
+    }
+
+    return false;
+  }
+
+ private:
+  //! The duration in seconds.
+  double duration;
+
+  //! Locally-stored timer object.
+  arma::wall_clock timer;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/traits.hpp

@@ -0,0 +1,416 @@
+/**
+ * @file traits.hpp
+ * @author Marcus Edel
+ *
+ * This file provides metaprogramming utilities for detecting certain members of
+ * CallbackType classes.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_TRAITS_HPP
+#define ENSMALLEN_CALLBACKS_TRAITS_HPP
+
+#include <ensmallen_bits/function/sfinae_utility.hpp>
+
+namespace ens {
+namespace callbacks {
+namespace traits {
+
+//! Detect an Evaluate() method.
+ENS_HAS_EXACT_METHOD_FORM(Evaluate, HasEvaluate)
+//! Detect an EvaluateConstraint() method.
+ENS_HAS_EXACT_METHOD_FORM(EvaluateConstraint, HasEvaluateConstraint)
+//! Detect an Gradient() method.
+ENS_HAS_EXACT_METHOD_FORM(Gradient, HasGradient)
+//! Detect an GradientConstraint() method.
+ENS_HAS_EXACT_METHOD_FORM(GradientConstraint, HasGradientConstraint)
+//! Detect an BeginOptimization() method.
+ENS_HAS_EXACT_METHOD_FORM(BeginOptimization, HasBeginOptimization)
+//! Detect an EndOptimization() method.
+ENS_HAS_EXACT_METHOD_FORM(EndOptimization, HasEndOptimization)
+//! Detect an BeginEpoch() method.
+ENS_HAS_EXACT_METHOD_FORM(BeginEpoch, HasBeginEpoch)
+//! Detect an EndEpoch() method.
+ENS_HAS_EXACT_METHOD_FORM(EndEpoch, HasEndEpoch)
+//! Detect an StepTaken() method.
+ENS_HAS_EXACT_METHOD_FORM(StepTaken, HasStepTaken)
+//! Detect an BatchSize() method.
+ENS_HAS_EXACT_METHOD_FORM(BatchSize, HasBatchSize)
+//! Detect an MaxIterations() method.
+ENS_HAS_EXACT_METHOD_FORM(MaxIterations, HasMaxIterations)
+//! Detect an NumFunctions() method.
+ENS_HAS_EXACT_METHOD_FORM(NumFunctions, HasNumFunctions)
+
+template<typename OptimizerType,
+         typename FunctionType,
+         typename MatType,
+         typename GradType = MatType>
+struct TypedForms
+{
+  //! This is the form of a bool Evaluate() callback method.
+  template<typename CallbackType>
+  using EvaluateBoolForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const double);
+
+  //! This is the form of a void Evaluate() callback method.
+  template<typename CallbackType>
+  using EvaluateVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const double);
+
+  //! This is the form of a bool EvaluateConstraint() callback method.
+  template<typename CallbackType>
+  using EvaluateConstraintBoolForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            const double);
+
+  //! This is the form of a void EvaluateConstraint() callback method.
+  template<typename CallbackType>
+  using EvaluateConstraintVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            const double);
+
+  //! This is the form of a bool Gradient() callback method.
+  template<typename CallbackType>
+  using GradientBoolForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const MatType&);
+
+  //! This is the form of a void Gradient() callback method.
+  template<typename CallbackType>
+  using GradientVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const MatType&);
+
+  //! This is the form of a bool GradientConstraint() callback method.
+  template<typename CallbackType>
+  using GradientConstraintBoolForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            const MatType&);
+
+  //! This is the form of a void GradientConstraint() callback method.
+  template<typename CallbackType>
+  using GradientConstraintVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            const MatType&);
+
+  //! This is the form of a bool BeginOptimization() callback method.
+  template<typename CallbackType>
+  using BeginOptimizationBoolForm =
+      bool(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            MatType&);
+
+  //! This is the form of a void BeginOptimization() callback method.
+  template<typename CallbackType>
+  using BeginOptimizationVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            MatType&);
+
+  //! This is the form of a bool EndOptimization() callback method.
+  template<typename CallbackType>
+  using EndOptimizationBoolForm =
+      bool(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            MatType&);
+
+  //! This is the form of a void EndOptimization() callback method.
+  template<typename CallbackType>
+  using EndOptimizationVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            MatType&);
+
+  //! This is the form of a bool BeginEpoch() callback method.
+  template<typename CallbackType>
+  using BeginEpochBoolForm =
+      bool(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            const double);
+
+  //! This is the form of a void BeginEpoch() callback method.
+  template<typename CallbackType>
+  using BeginEpochVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            const double);
+
+  //! This is the form of a bool EndEpoch() callback method.
+  template<typename CallbackType>
+  using EndEpochBoolForm =
+      bool(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            const double);
+
+  //! This is the form of a void EndEpoch() callback method.
+  template<typename CallbackType>
+  using EndEpochVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            const double);
+
+  //! This is the form of a bool StepTaken() callback method.
+  template<typename CallbackType>
+  using StepTakenBoolForm =
+      bool(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&);
+
+  //! This is the form of a void StepTaken() callback method.
+  template<typename CallbackType>
+  using StepTakenVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&);
+};
+
+//! Utility struct, check if either void BeginOptimization() or
+//! bool BeginOptimization() exists.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename MatType>
+struct HasBeginOptimizationSignature
+{
+  const static bool hasBool =
+      HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template BeginOptimizationBoolForm>::value &&
+      !HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template BeginOptimizationVoidForm>::value;
+
+  const static bool hasVoid =
+      !HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template BeginOptimizationBoolForm>::value &&
+      HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
+         FunctionType, MatType>::template BeginOptimizationVoidForm>::value;
+
+  const static bool hasNone =
+      !HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template BeginOptimizationBoolForm>::value &&
+      !HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
+         FunctionType, MatType>::template BeginOptimizationVoidForm>::value;
+};
+
+//! Utility struct, check if either void Evaluate() or bool Evaluate()
+//! exists.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename MatType>
+struct HasEvaluateSignature
+{
+  const static bool value =
+      HasEvaluate<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType>::template EvaluateBoolForm>::value ||
+      HasEvaluate<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType>::template EvaluateVoidForm>::value;
+};
+
+//! Utility struct, check if either void EvaluateConstraint() or
+//! bool EvaluateConstraint() exists.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename MatType>
+struct HasEvaluateConstraintSignature
+{
+  const static bool value =
+      HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType>::template EvaluateConstraintBoolForm>::value ||
+      HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType>::template EvaluateConstraintVoidForm>::value;
+};
+
+//! Utility struct, check if either void Gradient() or bool Gradient()
+//! exists.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename MatType,
+         typename Gradient>
+struct HasGradientSignature
+{
+  const static bool value =
+      HasGradient<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType, Gradient>::template GradientBoolForm>::value ||
+      HasGradient<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType, Gradient>::template GradientVoidForm>::value;
+};
+
+//! Utility struct, check if either void GradientConstraint() or
+//! bool GradientConstraint() exists.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename MatType,
+         typename Gradient>
+struct HasGradientConstraintSignature
+{
+  const static bool value =
+      HasGradientConstraint<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType,
+          Gradient>::template GradientConstraintBoolForm>::value ||
+      HasGradientConstraint<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType,
+          Gradient>::template GradientConstraintVoidForm>::value;
+};
+
+//! Utility struct, check if either void EndOptimization() or
+//! bool EndOptimization() exists.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename MatType>
+struct HasEndOptimizationSignature
+{
+  const static bool value =
+      HasEndOptimization<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType>::template EndOptimizationBoolForm>::value ||
+      HasEndOptimization<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType>::template EndOptimizationVoidForm>::value;
+};
+
+//! Utility struct, check if either void BeginEpoch() or bool BeginEpoch()
+//! exists.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename MatType>
+struct HasBeginEpochSignature
+{
+  const static bool value =
+      HasBeginEpoch<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType>::template BeginEpochBoolForm>::value ||
+      HasBeginEpoch<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType>::template BeginEpochVoidForm>::value;
+};
+
+//! Utility struct, check if either void EndEpoch() or bool EndEpoch()
+//! exists.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename MatType>
+struct HasEndEpochSignature
+{
+  const static bool hasBool =
+      HasEndEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EndEpochBoolForm>::value &&
+      !HasEndEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EndEpochVoidForm>::value;
+
+  const static bool hasVoid =
+      !HasEndEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EndEpochBoolForm>::value &&
+      HasEndEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EndEpochVoidForm>::value;
+
+  const static bool hasNone =
+      !HasEndEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EndEpochBoolForm>::value &&
+      !HasEndEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EndEpochVoidForm>::value;
+};
+
+//! Utility struct, check if either void StepTaken() or bool StepTaken() exists.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename MatType>
+struct HasStepTakenSignature
+{
+  const static bool hasBool =
+      HasStepTaken<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template StepTakenBoolForm>::value &&
+      !HasStepTaken<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template StepTakenVoidForm>::value;
+
+  const static bool hasVoid =
+      !HasStepTaken<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template StepTakenBoolForm>::value &&
+      HasStepTaken<CallbackType, TypedForms<OptimizerType,
+         FunctionType, MatType>::template StepTakenVoidForm>::value;
+
+  const static bool hasNone =
+      !HasStepTaken<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template StepTakenBoolForm>::value &&
+      !HasStepTaken<CallbackType, TypedForms<OptimizerType,
+         FunctionType, MatType>::template StepTakenVoidForm>::value;
+};
+
+//! Utility struct, check if size_t BatchSize() const exists.
+template<typename OptimizerType>
+struct HasBatchSizeSignature
+{
+  template<typename C>
+  using BatchSizeForm = size_t(C::*)(void) const;
+
+  const static bool value =
+      HasBatchSize<OptimizerType, BatchSizeForm>::value;
+};
+
+//! Utility struct, check if size_t MaxIterations() const exists.
+template<typename OptimizerType>
+struct HasMaxIterationsSignature
+{
+  template<typename C>
+  using HasMaxIterationsForm = size_t(C::*)(void) const;
+
+  const static bool value =
+      HasMaxIterations<OptimizerType, HasMaxIterationsForm>::value;
+};
+
+//! Utility struct, check if size_t NumFunctions() const or
+//! size_t NumFunctions() exists.
+template<typename OptimizerType>
+struct HasNumFunctionsSignature
+{
+  template<typename C>
+  using NumFunctionsConstForm = size_t(C::*)(void) const;
+
+  template<typename C>
+  using NumFunctionsForm = size_t(C::*)(void);
+
+  const static bool value =
+      HasNumFunctions<OptimizerType, NumFunctionsForm>::value ||
+      HasNumFunctions<OptimizerType, NumFunctionsConstForm>::value;
+};
+
+} // namespace traits
+} // namespace callbacks
+} // namespace ens
+
+#endif

include/ensmallen_bits/cmaes/cmaes.hpp

@@ -83,12 +83,19 @@ class CMAES
    * objective value is returned.
    *
    * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate);
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(DecomposableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks);
 
   //! Get the step size.
   size_t PopulationSize() const { return lambda; }

include/ensmallen_bits/cmaes/cmaes_impl.hpp

@@ -41,13 +41,24 @@ CMAES<SelectionPolicyType>::CMAES(const size_t lambda,
 
 //! Optimize the function (minimize).
 template<typename SelectionPolicyType>
-template<typename DecomposableFunctionType>
-double CMAES<SelectionPolicyType>::Optimize(
-    DecomposableFunctionType& function, arma::mat& iterate)
+template<typename DecomposableFunctionType,
+         typename MatType,
+         typename... CallbackTypes>
+typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
+    DecomposableFunctionType& function,
+    MatType& iterateIn,
+    CallbackTypes&&... callbacks)
 {
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
   // Make sure that we have the methods that we need.  Long name...
   traits::CheckNonDifferentiableDecomposableFunctionTypeAPI<
-      DecomposableFunctionType>();
+      DecomposableFunctionType, BaseMatType>();
+  RequireDenseFloatingPointType<BaseMatType>();
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
 
   // Find the number of functions to use.
   const size_t numFunctions = function.NumFunctions();
@@ -58,15 +69,15 @@ double CMAES<SelectionPolicyType>::Optimize(
 
   // Parent weights.
   const size_t mu = std::round(lambda / 2);
-  arma::vec w = std::log(mu + 0.5) - arma::log(
-    arma::linspace<arma::vec>(0, mu - 1, mu) + 1.0);
-  w /= arma::sum(w);
+  BaseMatType w = std::log(mu + 0.5) - arma::log(
+      arma::linspace<BaseMatType>(0, mu - 1, mu) + 1.0);
+  w /= arma::accu(w);
 
   // Number of effective solutions.
   const double muEffective = 1 / arma::accu(arma::pow(w, 2));
 
   // Step size control parameters.
-  arma::vec sigma(3);
+  BaseMatType sigma(3, 1); // sigma is vector-shaped.
   sigma(0) = 0.3 * (upperBound - lowerBound);
   const double cs = (muEffective + 2) / (iterate.n_elem + muEffective + 5);
   const double ds = 1 + cs + 2 * std::max(std::sqrt((muEffective - 1) /
@@ -86,139 +97,158 @@ double CMAES<SelectionPolicyType>::Optimize(
       muEffective) / (std::pow(iterate.n_elem + 2, 2) +
       alphaMu * muEffective / 2));
 
-  arma::cube mPosition(iterate.n_rows, iterate.n_cols, 3);
-  mPosition.slice(0) = lowerBound + arma::randu(
+  std::vector<BaseMatType> mPosition(3, BaseMatType(iterate.n_rows,
+      iterate.n_cols));
+  mPosition[0] = lowerBound + arma::randu<BaseMatType>(
       iterate.n_rows, iterate.n_cols) * (upperBound - lowerBound);
 
-  arma::mat step = arma::zeros(iterate.n_rows, iterate.n_cols);
+  BaseMatType step(iterate.n_rows, iterate.n_cols);
+  step.zeros();
 
   // Calculate the first objective function.
-  double currentObjective = 0;
+  ElemType currentObjective = 0;
   for (size_t f = 0; f < numFunctions; f += batchSize)
   {
     const size_t effectiveBatchSize = std::min(batchSize, numFunctions - f);
-    currentObjective += function.Evaluate(mPosition.slice(0), f,
+    const ElemType objective = function.Evaluate(mPosition[0], f,
         effectiveBatchSize);
+    currentObjective += objective;
+
+    Callback::Evaluate(*this, function, mPosition[0], objective,
+        callbacks...);
   }
 
-  double overallObjective = currentObjective;
-  double lastObjective = DBL_MAX;
+  ElemType overallObjective = currentObjective;
+  ElemType lastObjective = std::numeric_limits<ElemType>::max();
 
   // Population parameters.
-  arma::cube pStep(iterate.n_rows, iterate.n_cols, lambda);
-  arma::cube pPosition(iterate.n_rows, iterate.n_cols, lambda);
-  arma::vec pObjective(lambda);
-  arma::cube ps = arma::zeros(iterate.n_rows, iterate.n_cols, 2);
-  arma::cube pc = ps;
-  arma::cube C(iterate.n_elem, iterate.n_elem, 2);
-  C.slice(0).eye();
+  std::vector<BaseMatType> pStep(lambda, BaseMatType(iterate.n_rows,
+      iterate.n_cols));
+  std::vector<BaseMatType> pPosition(lambda, BaseMatType(iterate.n_rows,
+      iterate.n_cols));
+  BaseMatType pObjective(lambda, 1); // pObjective is vector-shaped.
+  std::vector<BaseMatType> ps(2, BaseMatType(iterate.n_rows, iterate.n_cols));
+  ps[0].zeros();
+  ps[1].zeros();
+  std::vector<BaseMatType> pc = ps;
+  std::vector<BaseMatType> C(2, BaseMatType(iterate.n_elem, iterate.n_elem));
+  C[0].eye();
 
   // Covariance matrix parameters.
-  arma::vec eigval;
-  arma::mat eigvec;
-  arma::vec eigvalZero = arma::zeros(iterate.n_elem);
+  arma::Col<ElemType> eigval; // TODO: might need a more general type.
+  BaseMatType eigvec;
+  BaseMatType eigvalZero(iterate.n_elem, 1); // eigvalZero is vector-shaped.
+  eigvalZero.zeros();
 
   // The current visitation order (sorted by population objectives).
   arma::uvec idx = arma::linspace<arma::uvec>(0, lambda - 1, lambda);
 
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
   // Now iterate!
-  for (size_t i = 1; i < maxIterations; ++i)
+  terminate |= Callback::BeginOptimization(*this, function, iterate,
+      callbacks...);
+  for (size_t i = 1; i < maxIterations && !terminate; ++i)
   {
     // To keep track of where we are.
     const size_t idx0 = (i - 1) % 2;
     const size_t idx1 = i % 2;
 
-    const arma::mat covLower = arma::chol(C.slice(idx0), "lower");
+    // Perform Cholesky decomposition. If the matrix is not positive definite,
+    // add a small value and try again.
+    BaseMatType covLower;
+    while (!arma::chol(covLower, C[idx0], "lower"))
+      C[idx0].diag() += 1e-16;
 
     for (size_t j = 0; j < lambda; ++j)
     {
       if (iterate.n_rows > iterate.n_cols)
       {
-        pStep.slice(idx(j)) = covLower *
-            arma::randn(iterate.n_rows, iterate.n_cols);
+        pStep[idx(j)] = covLower *
+            arma::randn<BaseMatType>(iterate.n_rows, iterate.n_cols);
       }
       else
       {
-        pStep.slice(idx(j)) = arma::randn(iterate.n_rows, iterate.n_cols) *
-            covLower;
+        pStep[idx(j)] = arma::randn<BaseMatType>(iterate.n_rows, iterate.n_cols)
+            * covLower;
       }
 
-      pPosition.slice(idx(j)) = mPosition.slice(idx0) + sigma(idx0) *
-          pStep.slice(idx(j));
+      pPosition[idx(j)] = mPosition[idx0] + sigma(idx0) * pStep[idx(j)];
 
       // Calculate the objective function.
       pObjective(idx(j)) = selectionPolicy.Select(function, batchSize,
-          pPosition.slice(idx(j)));
+          pPosition[idx(j)], callbacks...);
     }
 
     // Sort population.
-    idx = sort_index(pObjective);
+    idx = arma::sort_index(pObjective);
 
-    step = w(0) * pStep.slice(idx(0));
+    step = w(0) * pStep[idx(0)];
     for (size_t j = 1; j < mu; ++j)
-      step += w(j) * pStep.slice(idx(j));
+      step += w(j) * pStep[idx(j)];
 
-    mPosition.slice(idx1) = mPosition.slice(idx0) + sigma(idx0) * step;
+    mPosition[idx1] = mPosition[idx0] + sigma(idx0) * step;
 
     // Calculate the objective function.
     currentObjective = selectionPolicy.Select(function, batchSize,
-          mPosition.slice(idx1));
+        mPosition[idx1], callbacks...);
 
     // Update best parameters.
     if (currentObjective < overallObjective)
     {
       overallObjective = currentObjective;
-      iterate = mPosition.slice(idx1);
+      iterate = mPosition[idx1];
+
+      terminate |= Callback::StepTaken(*this, function, iterate, callbacks...);
     }
 
     // Update Step Size.
     if (iterate.n_rows > iterate.n_cols)
     {
-      ps.slice(idx1) = (1 - cs) * ps.slice(idx0) + std::sqrt(
+      ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt(
           cs * (2 - cs) * muEffective) * covLower.t() * step;
     }
     else
     {
-      ps.slice(idx1) = (1 - cs) * ps.slice(idx0) + std::sqrt(
+      ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt(
           cs * (2 - cs) * muEffective) * step * covLower.t();
     }
 
-    const double psNorm = arma::norm(ps.slice(idx1));
+    const ElemType psNorm = arma::norm(ps[idx1]);
     sigma(idx1) = sigma(idx0) * std::pow(
         std::exp(cs / ds * psNorm / enn - 1), 0.3);
 
     // Update covariance matrix.
     if ((psNorm / sqrt(1 - std::pow(1 - cs, 2 * i))) < h)
     {
-      pc.slice(idx1) = (1 - cc) * pc.slice(idx0) + std::sqrt(cc * (2 - cc) *
+      pc[idx1] = (1 - cc) * pc[idx0] + std::sqrt(cc * (2 - cc) *
         muEffective) * step;
 
-
       if (iterate.n_rows > iterate.n_cols)
       {
-        C.slice(idx1) = (1 - c1 - cmu) * C.slice(idx0) + c1 *
-          (pc.slice(idx1) * pc.slice(idx1).t());
+        C[idx1] = (1 - c1 - cmu) * C[idx0] + c1 *
+          (pc[idx1] * pc[idx1].t());
       }
       else
       {
-        C.slice(idx1) = (1 - c1 - cmu) * C.slice(idx0) + c1 *
-          (pc.slice(idx1).t() * pc.slice(idx1));
+        C[idx1] = (1 - c1 - cmu) * C[idx0] + c1 *
+          (pc[idx1].t() * pc[idx1]);
       }
     }
     else
     {
-      pc.slice(idx1) = (1 - cc) * pc.slice(idx0);
+      pc[idx1] = (1 - cc) * pc[idx0];
 
       if (iterate.n_rows > iterate.n_cols)
       {
-        C.slice(idx1) = (1 - c1 - cmu) * C.slice(idx0) + c1 * (pc.slice(idx1) *
-            pc.slice(idx1).t() + (cc * (2 - cc)) * C.slice(idx0));
+        C[idx1] = (1 - c1 - cmu) * C[idx0] + c1 * (pc[idx1] *
+            pc[idx1].t() + (cc * (2 - cc)) * C[idx0]);
       }
       else
       {
-        C.slice(idx1) = (1 - c1 - cmu) * C.slice(idx0) + c1 *
-            (pc.slice(idx1).t() * pc.slice(idx1) + (cc * (2 - cc)) *
-            C.slice(idx0));
+        C[idx1] = (1 - c1 - cmu) * C[idx0] + c1 *
+            (pc[idx1].t() * pc[idx1] + (cc * (2 - cc)) * C[idx0]);
       }
     }
 
@@ -226,30 +256,30 @@ double CMAES<SelectionPolicyType>::Optimize(
     {
       for (size_t j = 0; j < mu; ++j)
       {
-        C.slice(idx1) = C.slice(idx1) + cmu * w(j) *
-            pStep.slice(idx(j)) * pStep.slice(idx(j)).t();
+        C[idx1] = C[idx1] + cmu * w(j) *
+            pStep[idx(j)] * pStep[idx(j)].t();
       }
     }
     else
     {
       for (size_t j = 0; j < mu; ++j)
       {
-        C.slice(idx1) = C.slice(idx1) + cmu * w(j) *
-            pStep.slice(idx(j)).t() * pStep.slice(idx(j));
+        C[idx1] = C[idx1] + cmu * w(j) *
+            pStep[idx(j)].t() * pStep[idx(j)];
       }
     }
 
-    arma::eig_sym(eigval, eigvec, C.slice(idx1));
-    const arma::uvec negativeEigval = find(eigval < 0, 1);
+    arma::eig_sym(eigval, eigvec, C[idx1]);
+    const arma::uvec negativeEigval = arma::find(eigval < 0, 1);
     if (!negativeEigval.is_empty())
     {
       if (negativeEigval(0) == 0)
       {
-        C.slice(idx1).zeros();
+        C[idx1].zeros();
       }
       else
       {
-        C.slice(idx1) = eigvec.cols(0, negativeEigval(0) - 1) *
+        C[idx1] = eigvec.cols(0, negativeEigval(0) - 1) *
             arma::diagmat(eigval.subvec(0, negativeEigval(0) - 1)) *
             eigvec.cols(0, negativeEigval(0) - 1).t();
       }
@@ -263,6 +293,8 @@ double CMAES<SelectionPolicyType>::Optimize(
     {
       Warn << "CMA-ES: converged to " << overallObjective << "; "
           << "terminating with failure.  Try a smaller step size?" << std::endl;
+
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
       return overallObjective;
     }
 
@@ -270,12 +302,15 @@ double CMAES<SelectionPolicyType>::Optimize(
     {
       Info << "CMA-ES: minimized within tolerance " << tolerance << "; "
           << "terminating optimization." << std::endl;
+
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
       return overallObjective;
     }
 
     lastObjective = overallObjective;
   }
 
+  Callback::EndOptimization(*this, function, iterate, callbacks...);
   return overallObjective;
 }
 

include/ensmallen_bits/cmaes/full_selection.hpp

@@ -28,19 +28,24 @@ class FullSelection
    * @param batchSize Batch size to use for each step.
    * @param iterate starting point.
    */
-  template<typename DecomposableFunctionType>
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
   double Select(DecomposableFunctionType& function,
-                      const size_t batchSize,
-                      const arma::mat& iterate)
+                const size_t batchSize,
+                const MatType& iterate,
+                CallbackTypes&... callbacks)
   {
     // Find the number of functions to use.
     const size_t numFunctions = function.NumFunctions();
 
-    double objective = 0;
+    typename MatType::elem_type objective = 0;
     for (size_t f = 0; f < numFunctions; f += batchSize)
     {
       const size_t effectiveBatchSize = std::min(batchSize, numFunctions - f);
       objective += function.Evaluate(iterate, f, effectiveBatchSize);
+
+      Callback::Evaluate(*this, f, iterate, objective, callbacks...);
     }
 
     return objective;

include/ensmallen_bits/cmaes/random_selection.hpp

@@ -43,15 +43,18 @@ class RandomSelection
    * @param batchSize Batch size to use for each step.
    * @param iterate starting point.
    */
-  template<typename DecomposableFunctionType>
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
   double Select(DecomposableFunctionType& function,
-                      const size_t batchSize,
-                      const arma::mat& iterate)
+                const size_t batchSize,
+                const MatType& iterate,
+                CallbackTypes&... callbacks)
   {
     // Find the number of functions to use.
     const size_t numFunctions = function.NumFunctions();
 
-    double objective = 0;
+    typename MatType::elem_type objective = 0;
     for (size_t f = 0; f < std::floor(numFunctions * fraction); f += batchSize)
     {
       const size_t selection = arma::as_scalar(arma::randi<arma::uvec>(
@@ -60,6 +63,8 @@ class RandomSelection
           numFunctions - selection);
 
       objective += function.Evaluate(iterate, selection, effectiveBatchSize);
+
+      Callback::Evaluate(*this, f, iterate, objective, callbacks...);
     }
 
     return objective;

include/ensmallen_bits/cne/cne.hpp

@@ -56,8 +56,10 @@ namespace ens {
  * The whole process then repeats for multiple generation until at least one of
  * the termination criteria is met:
  *
- * 1) The final value of the objective function (Not considered if not provided).
- * 2) The maximum number of generation reached (optional but highly recommended).
+ * 1) The final value of the objective function (Not considered if not
+ *    provided).
+ * 2) The maximum number of generation reached (optional but highly
+ *    recommended).
  * 3) Minimum change in best fitness values between two consecutive generations
  *    should be greater than a threshold value (Not considered if not provided).
  *
@@ -87,30 +89,33 @@ class CNE
    *     the next generation.
    * @param tolerance The final value of the objective function for termination.
    *     If set to negative value, tolerance is not considered.
-   * @param objectiveChange Minimum change in best fitness values between two
-   *     consecutive generations should be greater than threshold. If set to
-   *     negative value, objectiveChange is not considered.
    */
   CNE(const size_t populationSize = 500,
       const size_t maxGenerations = 5000,
       const double mutationProb = 0.1,
       const double mutationSize = 0.02,
       const double selectPercent = 0.2,
-      const double tolerance = 1e-5,
-      const double objectiveChange = 1e-5);
+      const double tolerance = 1e-5);
 
   /**
    * Optimize the given function using CNE. The given
    * starting point will be modified to store the finishing point of the
    * algorithm, and the final objective value is returned.
    *
-   * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam ArbitraryFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate);
+  template<typename ArbitraryFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(ArbitraryFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks);
 
   //! Get the population size.
   size_t PopulationSize() const { return populationSize; }
@@ -137,22 +142,21 @@ class CNE
   //! Modify the selection percentage.
   double& SelectionPercentage() { return selectPercent; }
 
-  //! Get the final objective value.
+  //! Get the tolerance.
   double Tolerance() const { return tolerance; }
-  //! Modify the final objective value.
+  //! Modify the tolerance.
   double& Tolerance() { return tolerance; }
 
-  //! Get the change in fitness history between generations.
-  double ObjectiveChange() const { return objectiveChange; }
-  //! Modify the termination criteria of change in fitness value.
-  double& ObjectiveChange() { return objectiveChange; }
-
  private:
   //! Reproduce candidates to create the next generation.
-  void Reproduce();
+  template<typename MatType>
+  void Reproduce(std::vector<MatType>& population,
+                 const MatType& fitnessValues,
+                 arma::uvec& index);
 
   //! Modify weights with some noise for the evolution of next generation.
-  void Mutate();
+  template<typename MatType>
+  void Mutate(std::vector<MatType>& population, arma::uvec& index);
 
   /**
    * Crossover parents and create new childs. Two parents create two new childs.
@@ -166,20 +170,13 @@ class CNE
    *                 generation and place a child over there for the
    *                 next generation.
    */
-  void Crossover(const size_t mom,
+  template<typename MatType>
+  void Crossover(std::vector<MatType>& population,
+                 const size_t mom,
                  const size_t dad,
                  const size_t dropout1,
                  const size_t dropout2);
 
-  //! Population matrix. Each column is a candidate.
-  arma::cube population;
-
-  //! Vector of fintness values corresponding to each candidate.
-  arma::vec fitnessValues;
-
-  //! Index of sorted fitness values.
-  arma::uvec index;
-
   //! The number of candidates in the population.
   size_t populationSize;
 
@@ -198,9 +195,6 @@ class CNE
   //! The final value of the objective function.
   double tolerance;
 
-  //! Minimum change in best fitness values between two generations.
-  double objectiveChange;
-
   //! Number of candidates to become parent for the next generation.
   size_t numElite;
 

include/ensmallen_bits/cne/cne_impl.hpp

@@ -25,23 +25,39 @@ inline CNE::CNE(const size_t populationSize,
                 const double mutationProb,
                 const double mutationSize,
                 const double selectPercent,
-                const double tolerance,
-                const double objectiveChange) :
+                const double tolerance) :
     populationSize(populationSize),
     maxGenerations(maxGenerations),
     mutationProb(mutationProb),
     mutationSize(mutationSize),
     selectPercent(selectPercent),
     tolerance(tolerance),
-    objectiveChange(objectiveChange),
     numElite(0),
     elements(0)
 { /* Nothing to do here. */ }
 
 //! Optimize the function.
-template<typename DecomposableFunctionType>
-double CNE::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
+template<typename ArbitraryFunctionType,
+         typename MatType,
+         typename... CallbackTypes>
+typename MatType::elem_type CNE::Optimize(ArbitraryFunctionType& function,
+                                          MatType& iterateIn,
+                                          CallbackTypes&&... callbacks)
 {
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
+  // Make sure that we have the methods that we need.  Long name...
+  traits::CheckNonDifferentiableFunctionTypeAPI<ArbitraryFunctionType,
+      BaseMatType>();
+  RequireDenseFloatingPointType<BaseMatType>();
+
+  // Vector of fitness values corresponding to each candidate.
+  BaseMatType fitnessValues;
+  //! Index of sorted fitness values.
+  arma::uvec index;
+
   // Make sure for evolution to work at least four candidates are present.
   if (populationSize < 4)
   {
@@ -70,54 +86,64 @@ double CNE::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
         "children. Increase population size.");
   }
 
-  // Set the population size and fill random values [0,1].
-  population = arma::randu(iterate.n_rows, iterate.n_cols, populationSize);
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
 
-  // Store the number of elements in a cube slice or a matrix column.
-  elements = population.n_rows * population.n_cols;
+  // Generate the population based on a Gaussian distribution around the given
+  // starting point.
+  std::vector<BaseMatType> population;
+  for (size_t i = 0 ; i < populationSize; ++i)
+  {
+    population.push_back(arma::randu<BaseMatType>(iterate.n_rows,
+        iterate.n_cols) + iterate);
+  }
 
-  // initializing helper variables.
+  // Store the number of elements in the objective matrix.
+  elements = iterate.n_rows * iterate.n_cols;
+
+  // Initialize helper variables.
   fitnessValues.set_size(populationSize);
 
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
   Info << "CNE initialized successfully. Optimization started."
       << std::endl;
 
   // Find the fitness before optimization using given iterate parameters.
-  size_t lastBestFitness = function.Evaluate(iterate);
+  ElemType lastBestFitness = function.Evaluate(iterate);
+  Callback::Evaluate(*this, function, iterate, lastBestFitness, callbacks...);
 
   // Iterate until maximum number of generations is obtained.
-  for (size_t gen = 1; gen <= maxGenerations; gen++)
+  terminate |= Callback::BeginOptimization(*this, function, iterate,
+      callbacks...);
+  for (size_t gen = 1; gen <= maxGenerations && !terminate; gen++)
   {
     // Calculating fitness values of all candidates.
     for (size_t i = 0; i < populationSize; i++)
     {
        // Select a candidate and insert the parameters in the function.
-       iterate = population.slice(i);
+       iterate = population[i];
+       terminate |= Callback::StepTaken(*this, function, iterate,
+          callbacks...);
 
        // Find fitness of candidate.
        fitnessValues[i] = function.Evaluate(iterate);
+
+       Callback::Evaluate(*this, function, iterate, fitnessValues[i],
+          callbacks...);
     }
 
     Info << "Generation number: " << gen << " best fitness = "
         << fitnessValues.min() << std::endl;
 
     // Create next generation of species.
-    Reproduce();
+    Reproduce(population, fitnessValues, index);
 
     // Check for termination criteria.
-    if (tolerance >= fitnessValues.min())
+    if (std::abs(lastBestFitness - fitnessValues.min()) < tolerance)
     {
-      Info << "CNE::Optimize(): terminating. Given fitness criteria "
-          << tolerance << " > " << fitnessValues.min() << "." << std::endl;
-      break;
-    }
-
-    // Check for termination criteria.
-    if (lastBestFitness - fitnessValues.min() < objectiveChange)
-    {
-      Info << "CNE::Optimize(): terminating. Fitness history change "
-          << (lastBestFitness - fitnessValues.min())
-          << " < " << objectiveChange << "." << std::endl;
+      Info << "CNE: minimized within tolerance " << tolerance << "; "
+            << "terminating optimization." << std::endl;
       break;
     }
 
@@ -126,13 +152,20 @@ double CNE::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
   }
 
   // Set the best candidate into the network parameters.
-  iterate = population.slice(index(0));
+  iterateIn = population[index(0)];
 
-  return function.Evaluate(iterate);
+  const ElemType objective = function.Evaluate(iterate);
+  Callback::Evaluate(*this, function, iterate, objective, callbacks...);
+
+  Callback::EndOptimization(*this, function, iterate, callbacks...);
+  return objective;
 }
 
 //! Reproduce candidates to create the next generation.
-inline void CNE::Reproduce()
+template<typename MatType>
+inline void CNE::Reproduce(std::vector<MatType>& population,
+                           const MatType& fitnessValues,
+                           arma::uvec& index)
 {
   // Sort fitness values. Smaller fitness value means better performance.
   index = arma::sort_index(fitnessValues);
@@ -167,54 +200,56 @@ inline void CNE::Reproduce()
 
     // Parents generate 2 children replacing the dropped-out candidates.
     // Also finding the index of these candidates in the population matrix.
-    Crossover(index[mom], index[dad], index[i], index[i + 1]);
+    Crossover(population, index[mom], index[dad], index[i], index[i + 1]);
   }
 
   // Mutating the weights with small noise values.
   // This is done to bring change in the next generation.
-  Mutate();
+  Mutate(population, index);
 }
 
 //! Crossover parents to create new children.
-inline void CNE::Crossover(const size_t mom,
+template<typename MatType>
+inline void CNE::Crossover(std::vector<MatType>& population,
+                           const size_t mom,
                            const size_t dad,
                            const size_t child1,
                            const size_t child2)
 {
   // Replace the candidates with parents at their place.
-  population.slice(child1) = population.slice(mom);
-  population.slice(child2) = population.slice(dad);
-
-  // Preallocate random selection vector (values between 0 and 1).
-  arma::vec selection = arma::randu(elements);
+  population[child1] = population[mom];
+  population[child2] = population[dad];
 
   // Randomly alter mom and dad genome weights to get two different children.
   for (size_t i = 0; i < elements; i++)
   {
     // Using it to alter the weights of the children.
-    if (selection(i) > 0.5)
+    const double random = arma::randu<typename MatType::elem_type>();
+    if (random > 0.5)
     {
-      population.slice(child1)(i) = population.slice(mom)(i);
-      population.slice(child2)(i) = population.slice(dad)(i);
+      population[child1](i) = population[mom](i);
+      population[child2](i) = population[dad](i);
     }
     else
     {
-      population.slice(child1)(i) = population.slice(dad)(i);
-      population.slice(child2)(i) = population.slice(mom)(i);
+      population[child1](i) = population[dad](i);
+      population[child2](i) = population[mom](i);
     }
   }
 }
 
 //! Modify weights with some noise for the evolution of next generation.
-inline void CNE::Mutate()
+template<typename MatType>
+inline void CNE::Mutate(std::vector<MatType>& population, arma::uvec& index)
 {
   // Mutate the whole matrix with the given rate and probability.
   // The best candidate is not altered.
   for (size_t i = 1; i < populationSize; i++)
   {
-    population.slice(index(i)) += (arma::randu(
-        population.n_rows, population.n_cols) < mutationProb) %
-        (mutationSize * arma::randn(population.n_rows, population.n_cols));
+    population[index(i)] += (arma::randu<MatType>(population[index(i)].n_rows,
+        population[index(i)].n_cols) < mutationProb) %
+        (mutationSize * arma::randn<MatType>(population[index(i)].n_rows,
+        population[index(i)].n_cols));
   }
 }
 

include/ensmallen_bits/config.hpp

@@ -46,7 +46,26 @@
 #if defined(ENS_USE_OPENMP)
   #define ENS_PRAGMA_OMP_PARALLEL _Pragma("omp parallel")
   #define ENS_PRAGMA_OMP_ATOMIC   _Pragma("omp atomic")
+  #define ENS_PRAGMA_OMP_CRITICAL _Pragma("omp critical")
 #else
   #define ENS_PRAGMA_OMP_PARALLEL
   #define ENS_PRAGMA_OMP_ATOMIC
+  #define ENS_PRAGMA_OMP_CRITICAL
+#endif
+
+
+// Define ens_deprecated for deprecated functionality.
+// This is adapted from Armadillo's implementation.
+#if defined(_MSC_VER)
+  #define ens_deprecated __declspec(deprecated)
+#elif defined(__GNUG__) && (!defined(__clang__))
+  #define ens_deprecated __attribute__((__deprecated__))
+#elif defined(__clang__)
+  #if __has_attribute(__deprecated__)
+    #define ens_deprecated __attribute__((__deprecated__))
+  #else
+    #define ens_deprecated
+  #endif
+#else
+  #define ens_deprecated
 #endif

include/ensmallen_bits/de/de.hpp

@@ -0,0 +1,148 @@
+/**
+ * @file de.hpp
+ * @author Rahul Ganesh Prabhu
+ *
+ * Differential Evolution is a method used for global optimization of arbitrary
+ * functions that optimizes a problem by iteratively trying to improve a
+ * candidate solution.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_DE_DE_HPP
+#define ENSMALLEN_DE_DE_HPP
+
+namespace ens {
+
+/**
+ * Differential evolution is a stochastic evolutionary algorithm used for global
+ * optimization. This class implements the best/1/bin strategy of differential
+ * evolution to converge a given function to minima.
+ *
+ * The algorithm works by generating a fixed number of candidates from the
+ * given starting point. At each pass through the population, the algorithm
+ * mutates each candidate solution to create a trial solution. If the trial
+ * solution is better than the candidate, it is replaced in the
+ * population.
+ *
+ * The evolution takes place in two steps:
+ * - Mutation
+ * - Crossover
+ *
+ * Mutation is done by generating a new candidate solution from the best
+ * candidate of the previous solution and two random other candidates.
+ *
+ * Crossover is done by mixing the parameters of the candidate solution and the
+ * mutant solution. This is done only if a randomly generated number between 0
+ * and 1 is greater than the crossover rate.
+ *
+ * The final value and the parameters are returned by the Optimize() method.
+ *
+ * For more information, see the following:
+ *
+ * @techreport{storn1995,
+ *   title    = {Differential Evolution—a simple and efficient adaptive scheme
+ *               for global optimization over continuous spaces},
+ *   author   = {Storn, Rainer and Price, Kenneth},
+ *   year     = 1995
+ * }
+ *
+ * DE can optimize arbitrary functions.  For more details, see the
+ * documentation on function types included with this distribution or on the
+ * ensmallen website.
+ */
+class DE
+{
+ public:
+  /**
+   * Constructor for the DE optimizer
+   *
+   * The default values provided over here are not necessarily suitable for a
+   * given function. Therefore it is highly recommended to adjust the
+   * parameters according to the problem.
+   *
+   * @param populationSize The number of candidates in the population.
+   *     This should be at least 3 in size.
+   * @param maxGenerations The maximum number of generations allowed for CNE.
+   * @param crossoverRate  The probability that a crossover will occur.
+   * @param differentialWeight A parameter used in the mutation of candidate
+   *     solutions controls amplification factor of the differentiation.
+   * @param tolerance The final value of the objective function for termination.
+   */
+  DE(const size_t populationSize = 100,
+     const size_t maxGenerations = 2000,
+     const double crossoverRate = 0.6,
+     const double differentialWeight = 0.8,
+     const double tolerance = 1e-5);
+
+  /**
+   * Optimize the given function using DE. The given
+   * starting point will be modified to store the finishing point of the
+   * algorithm, and the final objective value is returned.
+   *
+   * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(DecomposableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks);
+
+  //! Get the population size.
+  size_t PopulationSize() const { return populationSize; }
+  //! Modify the population size.
+  size_t& PopulationSize() { return populationSize; }
+
+  //! Get maximum number of generations.
+  size_t MaxGenerations() const { return maxGenerations; }
+  //! Modify maximum number of generations.
+  size_t& MaxGenerations() { return maxGenerations; }
+
+  //! Get crossover rate.
+  double CrossoverRate() const { return crossoverRate; }
+  //! Modify crossover rate.
+  double& CrossoverRate() { return crossoverRate; }
+
+  //! Get differential weight.
+  double DifferentialWeight() const {return differentialWeight; }
+  //! Modify differential weight.
+  double& DifferentialWeight() { return differentialWeight; }
+
+  //! Get the tolerance.
+  double Tolerance() const { return tolerance; }
+  //! Modify the tolerance.
+  double& Tolerance() { return tolerance; }
+
+ private:
+  //! The number of candidates in the population.
+  size_t populationSize;
+
+  //! Maximum number of generations before termination criteria is met.
+  size_t maxGenerations;
+
+  //! Probability that crossover will occur.
+  double crossoverRate;
+
+  //! Amplification factor for differentiation.
+  double differentialWeight;
+
+  //! The tolerance for termination.
+  double tolerance;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "de_impl.hpp"
+
+#endif

include/ensmallen_bits/de/de_impl.hpp

@@ -0,0 +1,176 @@
+/**
+ * @file de_impl.hpp
+ * @author Rahul Ganesh Prabhu
+ *
+ * Implementation of Differential Evolution an evolutionary algorithm used for
+ * global optimization of arbitrary functions.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DE_DE_IMPL_HPP
+#define ENSMALLEN_DE_DE_IMPL_HPP
+
+#include "de.hpp"
+
+namespace ens {
+
+inline DE::DE(const size_t populationSize ,
+              const size_t maxGenerations,
+              const double crossoverRate,
+              const double differentialWeight,
+              const double tolerance):
+    populationSize(populationSize),
+    maxGenerations(maxGenerations),
+    crossoverRate(crossoverRate),
+    differentialWeight(differentialWeight),
+    tolerance(tolerance)
+{ /* Nothing to do here. */ }
+
+//!Optimize the function
+template<typename DecomposableFunctionType,
+         typename MatType,
+         typename... CallbackTypes>
+typename MatType::elem_type DE::Optimize(DecomposableFunctionType& function,
+                                         MatType& iterateIn,
+                                         CallbackTypes&&... callbacks)
+{
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  // Population matrix. Each column is a candidate.
+  std::vector<BaseMatType> population;
+  population.resize(populationSize);
+  // Vector of fitness values corresponding to each candidate.
+  arma::Col<ElemType> fitnessValues;
+
+  // Make sure that we have the methods that we need.  Long name...
+  traits::CheckNonDifferentiableDecomposableFunctionTypeAPI<
+      DecomposableFunctionType, BaseMatType>();
+  RequireDenseFloatingPointType<BaseMatType>();
+
+  // Population Size must be at least 3 for DE to work.
+  if (populationSize < 3)
+  {
+    throw std::logic_error("CNE::Optimize(): population size should be at least"
+        " 3!");
+  }
+
+  // Initialize helper variables.
+  fitnessValues.set_size(populationSize);
+  ElemType lastBestFitness = DBL_MAX;
+  BaseMatType bestElement;
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
+  // Generate a population based on a Gaussian distribution around the given
+  // starting point. Also finds the best element of the population.
+  for (size_t i = 0; i < populationSize; i++)
+  {
+    population[i].randn(iterate.n_rows, iterate.n_cols);
+    population[i] += iterate;
+    fitnessValues[i] = function.Evaluate(population[i]);
+
+    Callback::Evaluate(*this, function, population[i], fitnessValues[i],
+        callbacks...);
+
+    if (fitnessValues[i] < lastBestFitness)
+    {
+      lastBestFitness = fitnessValues[i];
+      bestElement = population[i];
+    }
+  }
+
+  // Iterate until maximum number of generations are completed.
+  terminate |= Callback::BeginOptimization(*this, function, iterate,
+      callbacks...);
+  for (size_t gen = 0; gen < maxGenerations && !terminate; gen++)
+  {
+    // Generate new population based on /best/1/bin strategy.
+    for (size_t member = 0; member < populationSize; member++)
+    {
+      iterate = population[member];
+
+      // Generate two different random numbers to choose two random members.
+      size_t l = 0, m = 0;
+      do
+      {
+        l = arma::randi<arma::uword>(arma::distr_param(0, populationSize - 1));
+      }
+      while (l == member);
+
+      do
+      {
+        m = arma::randi<arma::uword>(arma::distr_param(0, populationSize - 1));
+      }
+      while (m == member && m == l);
+
+      // Generate new "mutant" from two randomly chosen members.
+      BaseMatType mutant = bestElement + differentialWeight *
+          (population[l] - population[m]);
+
+      // Perform crossover.
+      const BaseMatType cr = arma::randu<BaseMatType>(iterate.n_rows);
+      for (size_t it = 0; it < iterate.n_rows; it++)
+      {
+        if (cr[it] >= crossoverRate)
+        {
+          mutant[it] = iterate[it];
+        }
+      }
+
+      ElemType iterateValue = function.Evaluate(iterate);
+      Callback::Evaluate(*this, function, iterate, iterateValue, callbacks...);
+
+      const ElemType mutantValue = function.Evaluate(mutant);
+      Callback::Evaluate(*this, function, mutant, mutantValue, callbacks...);
+
+      // Replace the current member if mutant is better.
+      if (mutantValue < iterateValue)
+      {
+        iterate = mutant;
+        iterateValue = mutantValue;
+
+        terminate |= Callback::StepTaken(*this, function, iterate,
+            callbacks...);
+      }
+
+      fitnessValues[member] = iterateValue;
+      population[member] = iterate;
+    }
+
+    // Check for termination criteria.
+    if (std::abs(lastBestFitness - fitnessValues.min()) < tolerance)
+    {
+      Info << "DE: minimized within tolerance " << tolerance << "; "
+          << "terminating optimization." << std::endl;
+      break;
+    }
+
+    // Update helper variables.
+    lastBestFitness = fitnessValues.min();
+    for (size_t it = 0; it < populationSize; it++)
+    {
+      if (fitnessValues[it] == lastBestFitness)
+      {
+        bestElement = population[it];
+        break;
+      }
+    }
+  }
+
+  iterate = bestElement;
+
+  Callback::EndOptimization(*this, function, iterate, callbacks...);
+  return lastBestFitness;
+}
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/ens_version.hpp

@@ -12,16 +12,16 @@
 
 // This follows the Semantic Versioning pattern defined in https://semver.org/.
 
-#define ENS_VERSION_MAJOR 1
+#define ENS_VERSION_MAJOR 2
 // The minor version is two digits so regular numerical comparisons of versions
 // work right.  The first minor version of a release is always 10.
-#define ENS_VERSION_MINOR 13
-#define ENS_VERSION_PATCH 2
+#define ENS_VERSION_MINOR 10
+#define ENS_VERSION_PATCH 0
 // If this is a release candidate, it will be reflected in the version name
 // (i.e. the version name will be "RC1", "RC2", etc.).  Otherwise the version
 // name will typically be a seemingly arbitrary set of words that does not
 // contain the capitalized string "RC".
-#define ENS_VERSION_NAME "Coronavirus Invasion"
+#define ENS_VERSION_NAME "Fried Chicken"
 
 namespace ens {
 

include/ensmallen_bits/eve/eve.hpp

@@ -72,6 +72,8 @@ class Eve
    * @param tolerance Maximum absolute tolerance to terminate algorithm.
    * @param shuffle If true, the function order is shuffled; otherwise, each
    *        function is visited in linear order.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
    */
   Eve(const double stepSize = 0.001,
       const size_t batchSize = 32,
@@ -82,7 +84,8 @@ class Eve
       const double clip = 10,
       const size_t maxIterations = 100000,
       const double tolerance = 1e-5,
-      const bool shuffle = true);
+      const bool shuffle = true,
+      const bool exactObjective = false);
 
   /**
    * Optimize the given function using stochastic gradient descent.  The given
@@ -90,12 +93,36 @@ class Eve
    * algorithm, and the final objective value is returned.
    *
    * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of the parameters matrix.
+   * @tparam GradType Type of the gradient matrix.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate);
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(DecomposableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(DecomposableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<DecomposableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
 
   //! Get the step size.
   double StepSize() const { return stepSize; }
@@ -147,6 +174,11 @@ class Eve
   //! Modify whether or not the individual functions are shuffled.
   bool& Shuffle() { return shuffle; }
 
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return exactObjective; }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return exactObjective; }
+
  private:
   //! The step size for each example.
   double stepSize;
@@ -178,6 +210,9 @@ class Eve
   //! Controls whether or not the individual functions are shuffled when
   //! iterating.
   bool shuffle;
+
+  //! Controls whether or not the actual Objective value is calculated.
+  bool exactObjective;
 };
 
 } // namespace ens

include/ensmallen_bits/eve/eve_impl.hpp

@@ -29,7 +29,8 @@ inline Eve::Eve(const double stepSize,
                 const double clip,
                 const size_t maxIterations,
                 const double tolerance,
-                const bool shuffle) :
+                const bool shuffle,
+                const bool exactObjective) :
     stepSize(stepSize),
     batchSize(batchSize),
     beta1(beta1),
@@ -39,75 +40,71 @@ inline Eve::Eve(const double stepSize,
     clip(clip),
     maxIterations(maxIterations),
     tolerance(tolerance),
-    shuffle(shuffle)
+    shuffle(shuffle),
+    exactObjective(exactObjective)
 { /* Nothing to do. */ }
 
 //! Optimize the function (minimize).
-template<typename DecomposableFunctionType>
-double Eve::Optimize(
-    DecomposableFunctionType& function,
-    arma::mat& iterate)
+template<typename DecomposableFunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value,
+typename MatType::elem_type>::type
+Eve::Optimize(DecomposableFunctionType& function,
+              MatType& iterateIn,
+              CallbackTypes&&... callbacks)
 {
-  typedef Function<DecomposableFunctionType> FullFunctionType;
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
+  typedef Function<DecomposableFunctionType, BaseMatType, BaseGradType>
+      FullFunctionType;
   FullFunctionType& f(static_cast<FullFunctionType&>(function));
 
   // Make sure we have all the methods that we need.
-  traits::CheckDecomposableFunctionTypeAPI<FullFunctionType>();
+  traits::CheckDecomposableFunctionTypeAPI<FullFunctionType, BaseMatType,
+      BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
 
   // Find the number of functions to use.
   const size_t numFunctions = f.NumFunctions();
 
   // To keep track of where we are and how things are going.
   size_t currentFunction = 0;
-  double overallObjective = 0;
-  double lastOverallObjective = DBL_MAX;
+  size_t epoch = 1;
+  ElemType overallObjective = 0;
+  ElemType lastOverallObjective = DBL_MAX;
 
-  double objective = 0;
-  double lastObjective = 0;
-  double dt = 1;
+  ElemType objective = 0;
+  ElemType lastObjective = 0;
+  ElemType dt = 1;
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
 
   // The exponential moving average of gradient values.
-  arma::mat m = arma::zeros<arma::mat>(iterate.n_rows, iterate.n_cols);
+  BaseGradType m(iterate.n_rows, iterate.n_cols);
+  m.zeros();
 
   // The exponential moving average of squared gradient values.
-  arma::mat v = arma::zeros<arma::mat>(iterate.n_rows, iterate.n_cols);
+  BaseGradType v(iterate.n_rows, iterate.n_cols);
+  v.zeros();
 
   // Now iterate!
-  arma::mat gradient(iterate.n_rows, iterate.n_cols);
+  terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  BaseGradType gradient(iterate.n_rows, iterate.n_cols);
   const size_t actualMaxIterations = (maxIterations == 0) ?
       std::numeric_limits<size_t>::max() : maxIterations;
-  for (size_t i = 0; i < actualMaxIterations; /* incrementing done manually */)
+  for (size_t i = 0; i < actualMaxIterations && !terminate;
+      /* incrementing done manually */)
   {
-    // Is this iteration the start of a sequence?
-    if ((currentFunction % numFunctions) == 0 && i > 0)
-    {
-      // Output current objective function.
-      Info << "Eve: iteration " << i << ", objective " << overallObjective
-          << "." << std::endl;
-
-      if (std::isnan(overallObjective) || std::isinf(overallObjective))
-      {
-        Warn << "Eve: converged to " << overallObjective << "; terminating"
-            << " with failure.  Try a smaller step size?" << std::endl;
-        return overallObjective;
-      }
-
-      if (std::abs(lastOverallObjective - overallObjective) < tolerance)
-      {
-        Info << "Eve: minimized within tolerance " << tolerance << "; "
-            << "terminating optimization." << std::endl;
-        return overallObjective;
-      }
-
-      // Reset the counter variables.
-      lastOverallObjective = overallObjective;
-      overallObjective = 0;
-      currentFunction = 0;
-
-      if (shuffle) // Determine order of visitation.
-        f.Shuffle();
-    }
-
     // Find the effective batch size; we have to take the minimum of three
     // things:
     // - the batch size can't be larger than the user-specified batch size;
@@ -124,6 +121,9 @@ double Eve::Optimize(
         gradient, effectiveBatchSize);
     overallObjective += objective;
 
+    terminate |= Callback::EvaluateWithGradient(*this, f, iterate,
+        objective, gradient, callbacks...);
+
     m *= beta1;
     m += (1 - beta1) * gradient;
 
@@ -135,11 +135,12 @@ double Eve::Optimize(
 
     if (i > 0)
     {
-      const double d = std::abs(objective - lastObjective) /
+      const ElemType d = std::abs(objective - lastObjective) /
           (std::min(objective, lastObjective) + epsilon);
 
       dt *= beta3;
-      dt += (1 - beta3) * std::min(std::max(d, 1.0 / clip), clip);
+      dt += (1 - beta3) * std::min(std::max(d, ElemType(1.0 / clip)),
+          ElemType(clip));
     }
 
     lastObjective = objective;
@@ -147,20 +148,69 @@ double Eve::Optimize(
     iterate -= stepSize / dt * (m / biasCorrection1) /
         (arma::sqrt(v / biasCorrection2) + epsilon);
 
+    terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
+
     i += effectiveBatchSize;
     currentFunction += effectiveBatchSize;
+
+    // Is this iteration the start of a sequence?
+    if ((currentFunction % numFunctions) == 0)
+    {
+      terminate |= Callback::EndEpoch(*this, f, iterate, epoch++,
+          overallObjective / (ElemType) numFunctions, callbacks...);
+
+      // Output current objective function.
+      Info << "Eve: iteration " << i << ", objective " << overallObjective
+          << "." << std::endl;
+
+      if (std::isnan(overallObjective) || std::isinf(overallObjective))
+      {
+        Warn << "Eve: converged to " << overallObjective << "; terminating"
+            << " with failure.  Try a smaller step size?" << std::endl;
+
+        Callback::EndOptimization(*this, f, iterate, callbacks...);
+        return overallObjective;
+      }
+
+      if (std::abs(lastOverallObjective - overallObjective) < tolerance ||
+          Callback::BeginEpoch(*this, f, iterate, epoch, overallObjective,
+              callbacks...))
+      {
+        Info << "Eve: minimized within tolerance " << tolerance << "; "
+            << "terminating optimization." << std::endl;
+
+        Callback::EndOptimization(*this, f, iterate, callbacks...);
+        return overallObjective;
+      }
+
+      // Reset the counter variables.
+      lastOverallObjective = overallObjective;
+      overallObjective = 0;
+      currentFunction = 0;
+
+      if (shuffle) // Determine order of visitation.
+        f.Shuffle();
+    }
   }
 
   Info << "Eve: maximum iterations (" << maxIterations << ") reached; "
       << "terminating optimization." << std::endl;
 
-  // Calculate final objective.
-  overallObjective = 0;
-  for (size_t i = 0; i < numFunctions; i += batchSize)
+  // Calculate final objective if exactObjective is set to true.
+  if (exactObjective)
   {
-    const size_t effectiveBatchSize = std::min(batchSize, numFunctions - i);
-    overallObjective += f.Evaluate(iterate, i, effectiveBatchSize);
+    overallObjective = 0;
+    for (size_t i = 0; i < numFunctions; i += batchSize)
+    {
+      const size_t effectiveBatchSize = std::min(batchSize, numFunctions - i);
+      const ElemType objective = f.Evaluate(iterate, i, effectiveBatchSize);
+      overallObjective += objective;
+
+      Callback::Evaluate(*this, f, iterate, objective, callbacks...);
+    }
   }
+
+  Callback::EndOptimization(*this, f, iterate, callbacks...);
   return overallObjective;
 }
 

include/ensmallen_bits/ftml/ftml.hpp

@@ -66,6 +66,8 @@ class FTML
    *        function is visited in linear order.
    * @param resetPolicy If true, parameters are reset before every Optimize
    *        call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
    */
   FTML(const double stepSize = 0.001,
        const size_t batchSize = 32,
@@ -75,7 +77,8 @@ class FTML
        const size_t maxIterations = 100000,
        const double tolerance = 1e-5,
        const bool shuffle = true,
-       const bool resetPolicy = true);
+       const bool resetPolicy = true,
+       const bool exactObjective = false);
 
   /**
    * Optimize the given function using FTML. The given starting point will
@@ -83,14 +86,39 @@ class FTML
    * objective value is returned.
    *
    * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate)
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(DecomposableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
   {
-    return optimizer.Optimize(function, iterate);
+    return optimizer.Optimize<DecomposableFunctionType, MatType, GradType,
+        CallbackTypes...>(function, iterate, callbacks...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(DecomposableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<DecomposableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
   }
 
   //! Get the step size.
@@ -133,6 +161,11 @@ class FTML
   //! Modify whether or not the individual functions are shuffled.
   bool& Shuffle() { return optimizer.Shuffle(); }
 
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
   //! Get whether or not the update policy parameters
   //! are reset before Optimize call.
   bool ResetPolicy() const { return optimizer.ResetPolicy(); }

include/ensmallen_bits/ftml/ftml_impl.hpp

@@ -25,7 +25,8 @@ inline FTML::FTML(const double stepSize,
                   const size_t maxIterations,
                   const double tolerance,
                   const bool shuffle,
-                  const bool resetPolicy) :
+                  const bool resetPolicy,
+                  const bool exactObjective) :
     optimizer(stepSize,
               batchSize,
               maxIterations,
@@ -33,7 +34,8 @@ inline FTML::FTML(const double stepSize,
               shuffle,
               FTMLUpdate(epsilon, beta1, beta2),
               NoDecay(),
-              resetPolicy)
+              resetPolicy,
+              exactObjective)
 { /* Nothing to do. */ }
 
 } // namespace ens

include/ensmallen_bits/ftml/ftml_update.hpp

@@ -34,7 +34,6 @@ namespace ens {
  * }
  * @endcode
  */
-
 class FTMLUpdate
 {
  public:
@@ -44,7 +43,7 @@ class FTMLUpdate
    * @param epsilon Epsilon is the minimum allowed gradient.
    * @param beta1 Exponential decay rate for the first moment estimates.
    * @param beta2 Exponential decay rate for the weighted infinity norm
-            estimates.
+   *        estimates.
    */
   FTMLUpdate(const double epsilon = 1e-8,
              const double beta1 = 0.9,
@@ -55,51 +54,6 @@ class FTMLUpdate
       iteration(0)
   { /* Do nothing. */ }
 
-  /**
-   * The Initialize method is called by SGD::Optimize method with UpdatePolicy
-   * FTMLUpdate before the start of the iteration update process.
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    v = arma::zeros<arma::mat>(rows, cols);
-    z = arma::zeros<arma::mat>(rows, cols);
-    d = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for FTML.
-   *
-   * @param iterate Parameter that minimizes the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    // Increment the iteration counter variable.
-    ++iteration;
-
-    // And update the iterate.
-    v *= beta2;
-    v += (1 - beta2) * (gradient % gradient);
-
-    const double biasCorrection1 = 1.0 - std::pow(beta1, iteration);
-    const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
-
-    arma::mat sigma = -beta1 * d;
-    d = biasCorrection1 / stepSize *
-      (arma::sqrt(v / biasCorrection2) + epsilon);
-    sigma += d;
-
-    z *= beta1;
-    z += (1 - beta1) * gradient - sigma % iterate;
-    iterate = -z / d;
-  }
-
   //! Get the value used to initialise the squared gradient parameter.
   double Epsilon() const { return epsilon; }
   //! Modify the value used to initialise the squared gradient parameter.
@@ -115,6 +69,84 @@ class FTMLUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent AdamUpdate object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(FTMLUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent)
+    {
+      v.zeros(rows, cols);
+      z.zeros(rows, cols);
+      d.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for FTML.
+     *
+     * @param iterate Parameter that minimizes the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // And update the iterate.
+      v *= parent.beta2;
+      v += (1 - parent.beta2) * (gradient % gradient);
+
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
+          parent.iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
+          parent.iteration);
+
+      MatType sigma = -parent.beta1 * d;
+      d = biasCorrection1 / stepSize *
+        (arma::sqrt(v / biasCorrection2) + parent.epsilon);
+      sigma += d;
+
+      z *= parent.beta1;
+      z += (1 - parent.beta1) * gradient - sigma % iterate;
+      iterate = -z / d;
+    }
+
+   private:
+    // Reference to instantiated parent object.
+    FTMLUpdate& parent;
+
+    // The exponential moving average of gradient values.
+    GradType v;
+
+    // The exponential moving average of squared gradient values.
+    GradType z;
+
+    // Parameter update term.
+    MatType d;
+  };
+
  private:
   // The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
@@ -125,17 +157,8 @@ class FTMLUpdate
   // The second moment coefficient.
   double beta2;
 
-  // The exponential moving average of gradient values.
-  arma::mat v;
-
-  // The exponential moving average of squared gradient values.
-  arma::mat z;
-
-  // Parmeter update term.
-  arma::mat d;
-
   // The number of iterations.
-  double iteration;
+  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/function.hpp

@@ -15,7 +15,7 @@
 
 namespace ens {
 
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 class Function;
 
 } // namespace ens
@@ -54,26 +54,28 @@ namespace ens {
  * addition, this class does not (to the best of my knowledge) rely on any
  * undefined behavior.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 class Function :
-    public AddDecomposableEvaluateWithGradientStatic<FunctionType>,
-    public AddDecomposableEvaluateWithGradientConst<FunctionType>,
-    public AddDecomposableEvaluateWithGradient<FunctionType>,
-    public AddDecomposableGradientStatic<FunctionType>,
-    public AddDecomposableGradientConst<FunctionType>,
-    public AddDecomposableGradient<FunctionType>,
-    public AddDecomposableEvaluateStatic<FunctionType>,
-    public AddDecomposableEvaluateConst<FunctionType>,
-    public AddDecomposableEvaluate<FunctionType>,
-    public AddEvaluateWithGradientStatic<FunctionType>,
-    public AddEvaluateWithGradientConst<FunctionType>,
-    public AddEvaluateWithGradient<FunctionType>,
-    public AddGradientStatic<FunctionType>,
-    public AddGradientConst<FunctionType>,
-    public AddGradient<FunctionType>,
-    public AddEvaluateStatic<FunctionType>,
-    public AddEvaluateConst<FunctionType>,
-    public AddEvaluate<FunctionType>,
+    public AddDecomposableEvaluateWithGradientStatic<FunctionType, MatType,
+        GradType>,
+    public AddDecomposableEvaluateWithGradientConst<FunctionType, MatType,
+        GradType>,
+    public AddDecomposableEvaluateWithGradient<FunctionType, MatType, GradType>,
+    public AddDecomposableGradientStatic<FunctionType, MatType, GradType>,
+    public AddDecomposableGradientConst<FunctionType, MatType, GradType>,
+    public AddDecomposableGradient<FunctionType, MatType, GradType>,
+    public AddDecomposableEvaluateStatic<FunctionType, MatType, GradType>,
+    public AddDecomposableEvaluateConst<FunctionType, MatType, GradType>,
+    public AddDecomposableEvaluate<FunctionType, MatType, GradType>,
+    public AddEvaluateWithGradientStatic<FunctionType, MatType, GradType>,
+    public AddEvaluateWithGradientConst<FunctionType, MatType, GradType>,
+    public AddEvaluateWithGradient<FunctionType, MatType, GradType>,
+    public AddGradientStatic<FunctionType, MatType, GradType>,
+    public AddGradientConst<FunctionType, MatType, GradType>,
+    public AddGradient<FunctionType, MatType, GradType>,
+    public AddEvaluateStatic<FunctionType, MatType, GradType>,
+    public AddEvaluateConst<FunctionType, MatType, GradType>,
+    public AddEvaluate<FunctionType, MatType, GradType>,
     public FunctionType
 {
  public:
@@ -82,25 +84,28 @@ class Function :
   // declarations here to ensure that they are all accessible.  Since we don't
   // know what FunctionType has, we can't use any using declarations there.
   using AddDecomposableEvaluateWithGradientStatic<
-      FunctionType>::EvaluateWithGradient;
+      FunctionType, MatType, GradType>::EvaluateWithGradient;
   using AddDecomposableEvaluateWithGradientConst<
-      FunctionType>::EvaluateWithGradient;
-  using AddDecomposableEvaluateWithGradient<FunctionType>::EvaluateWithGradient;
-  using AddDecomposableGradientStatic<FunctionType>::Gradient;
-  using AddDecomposableGradientConst<FunctionType>::Gradient;
-  using AddDecomposableGradient<FunctionType>::Gradient;
-  using AddDecomposableEvaluateStatic<FunctionType>::Evaluate;
-  using AddDecomposableEvaluateConst<FunctionType>::Evaluate;
-  using AddDecomposableEvaluate<FunctionType>::Evaluate;
-  using AddEvaluateWithGradientStatic<FunctionType>::EvaluateWithGradient;
-  using AddEvaluateWithGradientConst<FunctionType>::EvaluateWithGradient;
-  using AddEvaluateWithGradient<FunctionType>::EvaluateWithGradient;
-  using AddGradientStatic<FunctionType>::Gradient;
-  using AddGradientConst<FunctionType>::Gradient;
-  using AddGradient<FunctionType>::Gradient;
-  using AddEvaluateStatic<FunctionType>::Evaluate;
-  using AddEvaluateConst<FunctionType>::Evaluate;
-  using AddEvaluate<FunctionType>::Evaluate;
+      FunctionType, MatType, GradType>::EvaluateWithGradient;
+  using AddDecomposableEvaluateWithGradient<
+      FunctionType, MatType, GradType>::EvaluateWithGradient;
+  using AddDecomposableGradientStatic<
+      FunctionType, MatType, GradType>::Gradient;
+  using AddDecomposableGradientConst<FunctionType, MatType, GradType>::Gradient;
+  using AddDecomposableGradient<FunctionType, MatType, GradType>::Gradient;
+  using AddDecomposableEvaluateStatic<
+      FunctionType, MatType, GradType>::Evaluate;
+  using AddDecomposableEvaluateConst<FunctionType, MatType, GradType>::Evaluate;
+  using AddDecomposableEvaluate<FunctionType, MatType, GradType>::Evaluate;
+  using AddEvaluateWithGradientStatic<FunctionType, MatType, GradType>::EvaluateWithGradient;
+  using AddEvaluateWithGradientConst<FunctionType, MatType, GradType>::EvaluateWithGradient;
+  using AddEvaluateWithGradient<FunctionType, MatType, GradType>::EvaluateWithGradient;
+  using AddGradientStatic<FunctionType, MatType, GradType>::Gradient;
+  using AddGradientConst<FunctionType, MatType, GradType>::Gradient;
+  using AddGradient<FunctionType, MatType, GradType>::Gradient;
+  using AddEvaluateStatic<FunctionType, MatType, GradType>::Evaluate;
+  using AddEvaluateConst<FunctionType, MatType, GradType>::Evaluate;
+  using AddEvaluate<FunctionType, MatType, GradType>::Evaluate;
 };
 
 } // namespace ens

include/ensmallen_bits/function/add_decomposable_evaluate.hpp

@@ -23,35 +23,48 @@ namespace ens {
  * otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasDecomposableEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientForm>::value,
+                 traits::TypedForms<MatType, GradType>::template
+                     DecomposableEvaluateWithGradientForm
+             >::value,
          bool HasDecomposableEvaluate =
              traits::HasEvaluate<FunctionType,
-                 traits::DecomposableEvaluateForm>::value>
+                 traits::TypedForms<MatType, GradType>::template
+                      DecomposableEvaluateForm>::value>
 class AddDecomposableEvaluate
 {
  public:
   // Provide a dummy overload so the name 'Evaluate' exists for this object.
-  double Evaluate(traits::UnconstructableType&, const size_t, const size_t);
+  typename MatType::elem_type Evaluate(traits::UnconstructableType&,
+                                       const size_t,
+                                       const size_t);
 };
 
 /**
  * Reflect the existing Evaluate().
  */
-template<typename FunctionType, bool HasDecomposableEvaluateWithGradient>
-class AddDecomposableEvaluate<FunctionType, HasDecomposableEvaluateWithGradient,
-    true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasDecomposableEvaluateWithGradient>
+class AddDecomposableEvaluate<FunctionType, MatType, GradType,
+    HasDecomposableEvaluateWithGradient, true>
 {
  public:
   // Reflect the existing Evaluate().
-  double Evaluate(const arma::mat& coordinates,
-                  const size_t begin,
-                  const size_t batchSize)
+  typename MatType::elem_type Evaluate(const MatType& coordinates,
+                                       const size_t begin,
+                                       const size_t batchSize)
   {
     return static_cast<FunctionType*>(
-        static_cast<Function<FunctionType>*>(this))->Evaluate(coordinates,
-        begin, batchSize);
+        static_cast<Function<FunctionType,
+                             MatType,
+                             GradType>*>(this))->Evaluate(coordinates,
+                                                          begin,
+                                                          batchSize);
   }
 };
 
@@ -59,8 +72,8 @@ class AddDecomposableEvaluate<FunctionType, HasDecomposableEvaluateWithGradient,
  * If we have a decomposable EvaluateWithGradient() but not a decomposable
  * Evaluate(), add a decomposable Evaluate() method.
  */
-template<typename FunctionType>
-class AddDecomposableEvaluate<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddDecomposableEvaluate<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -71,12 +84,14 @@ class AddDecomposableEvaluate<FunctionType, true, false>
    * @param begin Index of first function to evaluate.
    * @param batchSize Number of functions to evaluate.
    */
-  double Evaluate(const arma::mat& coordinates,
+  double Evaluate(const MatType& coordinates,
                   const size_t begin,
                   const size_t batchSize)
   {
-    arma::mat gradient; // This will be ignored.
-    return static_cast<Function<FunctionType>*>(this)->EvaluateWithGradient(
+    GradType gradient; // This will be ignored.
+    return static_cast<Function<FunctionType,
+                                MatType,
+                                GradType>*>(this)->EvaluateWithGradient(
         coordinates, begin, gradient, batchSize);
   }
 };
@@ -87,36 +102,47 @@ class AddDecomposableEvaluate<FunctionType, true, false>
  * exists, or nothing otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasDecomposableEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientConstForm>::value,
+                 traits::TypedForms<MatType, GradType>::template
+                     DecomposableEvaluateWithGradientConstForm>::value,
          bool HasDecomposableEvaluate =
              traits::HasEvaluate<FunctionType,
-                 traits::DecomposableEvaluateConstForm>::value>
+                 traits::TypedForms<MatType, GradType>::template
+                     DecomposableEvaluateConstForm>::value>
 class AddDecomposableEvaluateConst
 {
  public:
   // Provide a dummy overload so the name 'Evaluate' exists for this object.
-  double Evaluate(traits::UnconstructableType&, const size_t, const size_t)
-      const;
+  typename MatType::elem_type Evaluate(traits::UnconstructableType&,
+                                       const size_t,
+                                       const size_t) const;
 };
 
 /**
  * Reflect the existing Evaluate().
  */
-template<typename FunctionType, bool HasDecomposableEvaluateWithGradient>
-class AddDecomposableEvaluateConst<FunctionType,
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasDecomposableEvaluateWithGradient>
+class AddDecomposableEvaluateConst<FunctionType, MatType, GradType,
     HasDecomposableEvaluateWithGradient, true>
 {
  public:
   // Reflect the existing Evaluate().
-  double Evaluate(const arma::mat& coordinates,
-                  const size_t begin,
-                  const size_t batchSize) const
+  typename MatType::elem_type Evaluate(const MatType& coordinates,
+                                       const size_t begin,
+                                       const size_t batchSize) const
   {
     return static_cast<const FunctionType*>(
-        static_cast<const Function<FunctionType>*>(this))->Evaluate(coordinates,
-        begin, batchSize);
+        static_cast<const Function<FunctionType,
+                                   MatType,
+                                   GradType>*>(this))->Evaluate(coordinates,
+                                                                begin,
+                                                                batchSize);
   }
 };
 
@@ -124,8 +150,8 @@ class AddDecomposableEvaluateConst<FunctionType,
  * If we have a decomposable const EvaluateWithGradient() but not a decomposable
  * const Evaluate(), add a decomposable const Evaluate() method.
  */
-template<typename FunctionType>
-class AddDecomposableEvaluateConst<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddDecomposableEvaluateConst<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -136,13 +162,14 @@ class AddDecomposableEvaluateConst<FunctionType, true, false>
    * @param begin Index of first function to evaluate.
    * @param batchSize Number of functions to evaluate.
    */
-  double Evaluate(const arma::mat& coordinates,
-                  const size_t begin,
-                  const size_t batchSize) const
+  typename MatType::elem_type Evaluate(const MatType& coordinates,
+                                       const size_t begin,
+                                       const size_t batchSize) const
   {
-    arma::mat gradient; // This will be ignored.
-    return
-        static_cast<const Function<FunctionType>*>(this)->EvaluateWithGradient(
+    GradType gradient; // This will be ignored.
+    return static_cast<const Function<FunctionType,
+                                      MatType,
+                                      GradType>*>(this)->EvaluateWithGradient(
         coordinates, begin, gradient, batchSize);
   }
 };
@@ -153,31 +180,40 @@ class AddDecomposableEvaluateConst<FunctionType, true, false>
  * exists, or nothing otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasDecomposableEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientStaticForm>::value,
+                 traits::TypedForms<MatType, GradType>::template
+                     DecomposableEvaluateWithGradientStaticForm>::value,
          bool HasDecomposableEvaluate =
              traits::HasEvaluate<FunctionType,
-                 traits::DecomposableEvaluateStaticForm>::value>
+                 traits::TypedForms<MatType, GradType>::template
+                     DecomposableEvaluateStaticForm>::value>
 class AddDecomposableEvaluateStatic
 {
  public:
   // Provide a dummy overload so the name 'Evaluate' exists for this object.
-  double Evaluate(traits::UnconstructableType&, const size_t) const;
+  static typename MatType::elem_type Evaluate(traits::UnconstructableType&,
+                                              const size_t,
+                                              const size_t);
 };
 
 /**
  * Reflect the existing Evaluate().
  */
-template<typename FunctionType, bool HasDecomposableEvaluateWithGradient>
-class AddDecomposableEvaluateStatic<FunctionType,
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasDecomposableEvaluateWithGradient>
+class AddDecomposableEvaluateStatic<FunctionType, MatType, GradType,
     HasDecomposableEvaluateWithGradient, true>
 {
  public:
   // Reflect the existing Evaluate().
-  static double Evaluate(const arma::mat& coordinates,
-                         const size_t begin,
-                         const size_t batchSize)
+  static typename MatType::elem_type Evaluate(const MatType& coordinates,
+                                              const size_t begin,
+                                              const size_t batchSize)
   {
     return FunctionType::Evaluate(coordinates, begin, batchSize);
   }
@@ -187,8 +223,9 @@ class AddDecomposableEvaluateStatic<FunctionType,
  * If we have a decomposable EvaluateWithGradient() but not a decomposable
  * Evaluate(), add a decomposable Evaluate() method.
  */
-template<typename FunctionType>
-class AddDecomposableEvaluateStatic<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddDecomposableEvaluateStatic<FunctionType, MatType, GradType, true,
+    false>
 {
  public:
   /**
@@ -199,11 +236,11 @@ class AddDecomposableEvaluateStatic<FunctionType, true, false>
    * @param begin Index of first function to evaluate.
    * @param batchSize Number of functions to evaluate.
    */
-  static double Evaluate(const arma::mat& coordinates,
-                         const size_t begin,
-                         const size_t batchSize)
+  static typename MatType::elem_type Evaluate(const MatType& coordinates,
+                                              const size_t begin,
+                                              const size_t batchSize)
   {
-    arma::mat gradient; // This will be ignored.
+    GradType gradient; // This will be ignored.
     return FunctionType::EvaluateWithGradient(coordinates, begin, gradient,
         batchSize);
   }

include/ensmallen_bits/function/add_decomposable_evaluate_const.hpp

@@ -1,165 +0,0 @@
-/**
- * @file add_decomposable_evaluate.hpp
- * @author Ryan Curtin
- *
- * Adds a decomposable Evaluate() function if a decomposable
- * EvaluateWithGradient() function exists.
- *
- * ensmallen is free software; you may redistribute it and/or modify it under
- * the terms of the 3-clause BSD license.  You should have received a copy of
- * the 3-clause BSD license along with ensmallen.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#ifndef ENSMALLEN_FUNCTION_ADD_DECOMPOSABLE_EVALUATE_HPP
-#define ENSMALLEN_FUNCTION_ADD_DECOMPOSABLE_EVALUATE_HPP
-
-#include "traits.hpp"
-
-namespace ens {
-
-/**
- * The AddDecomposableEvaluate mixin class will add a decomposable Evaluate()
- * method if a decomposable EvaluateWithGradient() function exists, or nothing
- * otherwise.
- */
-template<typename FunctionType,
-         bool HasDecomposableEvaluateWithGradient =
-             traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientForm>::value,
-         bool HasDecomposableEvaluate =
-             traits::HasEvaluate<FunctionType,
-                 traits::DecomposableEvaluateForm>::value>
-class AddDecomposableEvaluate
-{
- public:
-  // Provide a dummy overload so the name 'Evaluate' exists for this object.
-  double Evaluate(traits::UnconstructableType&, const size_t);
-};
-
-/**
- * Reflect the existing Evaluate().
- */
-template<typename FunctionType, bool HasDecomposableEvaluateWithGradient>
-class AddDecomposableEvaluate<FunctionType, HasDecomposableEvaluateWithGradient,
-    true>
-{
- public:
-  // Reflect the existing Evaluate().
-  double Evaluate(const arma::mat& coordinates, const size_t index)
-  {
-    return static_cast<FunctionType*>(
-        static_cast<Function<FunctionType>*>(this))->Evaluate(coordinates,
-        index);
-  }
-};
-
-/**
- * If we have a decomposable EvaluateWithGradient() but not a decomposable
- * Evaluate(), add a decomposable Evaluate() method.
- */
-template<typename FunctionType>
-class AddDecomposableEvaluate<FunctionType, true, false>
-{
- public:
-  /**
-   * Return the objective function for the given coordinates, starting at the
-   * given decomposable function using the given batch size.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param begin Index of first function to evaluate.
-   * @param batchSize Number of functions to evaluate.
-   */
-  double Evaluate(const arma::mat& coordinates,
-                  const size_t begin,
-                  const size_t batchSize)
-  {
-    arma::mat gradient; // This will be ignored.
-    return static_cast<Function<FunctionType>*>(this)->EvaluateWithGradient(
-        coordinates, begin, gradient, batchSize);
-  }
-};
-
-/**
- * The AddDecomposableEvaluateConst mixin class will add a decomposable const
- * Evaluate() method if a decomposable const EvaluateWithGradient() function
- * exists, or nothing otherwise.
- */
-template<typename FunctionType,
-         bool HasDecomposableEvaluateWithGradient =
-             traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientConstForm>::value,
-         bool HasDecomposableEvaluate =
-             traits::HasEvaluate<FunctionType,
-                 traits::DecomposableEvaluateConstForm>::value>
-class AddDecomposableEvaluateConst { };
-
-/**
- * If we have a decomposable const EvaluateWithGradient() but not a decomposable
- * const Evaluate(), add a decomposable const Evaluate() method.
- */
-template<typename FunctionType>
-class AddDecomposableEvaluateConst<FunctionType, true, false>
-{
- public:
-  /**
-   * Return the objective function for the given coordinates, starting at the
-   * given decomposable function using the given batch size.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param begin Index of first function to evaluate.
-   * @param batchSize Number of functions to evaluate.
-   */
-  double Evaluate(const arma::mat& coordinates,
-                  const size_t begin,
-                  const size_t batchSize) const
-  {
-    arma::mat gradient; // This will be ignored.
-    return
-        static_cast<const Function<FunctionType>*>(this)->EvaluateWithGradient(
-        coordinates, begin, gradient, batchSize);
-  }
-};
-
-/**
- * The AddDecomposableEvaluateStatic mixin class will add a decomposable static
- * Evaluate() method if a decomposable static EvaluateWithGradient() function
- * exists, or nothing otherwise.
- */
-template<typename FunctionType,
-         bool HasDecomposableEvaluateWithGradient =
-             traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientStaticForm>::value,
-         bool HasDecomposableEvaluate =
-             traits::HasEvaluate<FunctionType,
-                 traits::DecomposableEvaluateStaticForm>::value>
-class AddDecomposableEvaluateStatic { };
-
-/**
- * If we have a decomposable EvaluateWithGradient() but not a decomposable
- * Evaluate(), add a decomposable Evaluate() method.
- */
-template<typename FunctionType>
-class AddDecomposableEvaluateStatic<FunctionType, true, false>
-{
- public:
-  /**
-   * Return the objective function for the given coordinates, starting at the
-   * given decomposable function using the given batch size.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param begin Index of first function to evaluate.
-   * @param batchSize Number of functions to evaluate.
-   */
-  static double Evaluate(const arma::mat& coordinates,
-                         const size_t begin,
-                         const size_t batchSize)
-  {
-    arma::mat gradient; // This will be ignored.
-    return FunctionType::EvaluateWithGradient(coordinates, begin, gradient,
-        batchSize);
-  }
-};
-
-} // namespace ens
-
-#endif

include/ensmallen_bits/function/add_decomposable_evaluate_with_gradient.hpp

@@ -23,45 +23,61 @@ namespace ens {
  * decomposable Gradient() method exists, or nothing otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          // Check if there is at least one non-const Evaluate() or Gradient().
          bool HasDecomposableEvaluateGradient = traits::HasNonConstSignatures<
              FunctionType,
              traits::HasEvaluate,
-             traits::DecomposableEvaluateForm,
-             traits::DecomposableEvaluateConstForm,
-             traits::DecomposableEvaluateStaticForm,
+             traits::TypedForms<MatType, GradType>::template
+                 DecomposableEvaluateForm,
+             traits::TypedForms<MatType, GradType>::template
+                 DecomposableEvaluateConstForm,
+             traits::TypedForms<MatType, GradType>::template
+                 DecomposableEvaluateStaticForm,
              traits::HasGradient,
-             traits::DecomposableGradientForm,
-             traits::DecomposableGradientConstForm,
-             traits::DecomposableGradientStaticForm>::value,
+             traits::TypedForms<MatType, GradType>::template
+                 DecomposableGradientForm,
+             traits::TypedForms<MatType, GradType>::template
+                 DecomposableGradientConstForm,
+             traits::TypedForms<MatType, GradType>::template
+                 DecomposableGradientStaticForm>::value,
          bool HasDecomposableEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientForm>::value>
+                 traits::TypedForms<MatType, GradType>::template
+                     DecomposableEvaluateWithGradientForm>::value>
 class AddDecomposableEvaluateWithGradient
 {
  public:
   // Provide a dummy overload so the name 'EvaluateWithGradient' exists for this
   // object.
-  double EvaluateWithGradient(traits::UnconstructableType&, const size_t,
+  typename MatType::elem_type EvaluateWithGradient(
+      traits::UnconstructableType&,
+      const size_t,
       const size_t);
 };
 
 /**
  * Reflect the existing EvaluateWithGradient().
  */
-template<typename FunctionType, bool HasDecomposableEvaluateGradient>
-class AddDecomposableEvaluateWithGradient<FunctionType,
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasDecomposableEvaluateGradient>
+class AddDecomposableEvaluateWithGradient<FunctionType, MatType, GradType,
     HasDecomposableEvaluateGradient, true>
 {
  public:
-  // Reflect the existing Evaluate().
-  double EvaluateWithGradient(const arma::mat& coordinates,
-                              const size_t begin,
-                              arma::mat& gradient,
-                              const size_t batchSize)
+  // Reflect the existing EvaluateWithGradient().
+  typename MatType::elem_type EvaluateWithGradient(const MatType& coordinates,
+                                                   const size_t begin,
+                                                   GradType& gradient,
+                                                   const size_t batchSize)
   {
     return static_cast<FunctionType*>(
-        static_cast<Function<FunctionType>*>(this))->EvaluateWithGradient(
+        static_cast<Function<FunctionType,
+                             MatType,
+                             GradType>*>(this))->EvaluateWithGradient(
         coordinates, begin, gradient, batchSize);
   }
 };
@@ -71,8 +87,9 @@ class AddDecomposableEvaluateWithGradient<FunctionType,
  * not a decomposable EvaluateWithGradient(), add a decomposable
  * EvaluateWithGradient() method.
  */
-template<typename FunctionType>
-class AddDecomposableEvaluateWithGradient<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddDecomposableEvaluateWithGradient<FunctionType, MatType, GradType, true,
+    false>
 {
  public:
   /**
@@ -85,16 +102,16 @@ class AddDecomposableEvaluateWithGradient<FunctionType, true, false>
    * @param gradient Matrix to store the gradient into.
    * @param batchSize Number of decomposable functions to evaluate.
    */
-  double EvaluateWithGradient(const arma::mat& coordinates,
-                              const size_t begin,
-                              arma::mat& gradient,
-                              const size_t batchSize)
+  typename MatType::elem_type EvaluateWithGradient(const MatType& coordinates,
+                                                   const size_t begin,
+                                                   GradType& gradient,
+                                                   const size_t batchSize)
   {
-    const double objective =
-        static_cast<Function<FunctionType>*>(this)->Evaluate(coordinates, begin,
-        batchSize);
-    static_cast<Function<FunctionType>*>(this)->Gradient(coordinates, begin,
-        gradient, batchSize);
+    const typename MatType::elem_type objective =
+        static_cast<Function<FunctionType, MatType, GradType>*>(this)->Evaluate(
+        coordinates, begin, batchSize);
+    static_cast<Function<FunctionType, MatType, GradType>*>(this)->Gradient(
+        coordinates, begin, gradient, batchSize);
     return objective;
   }
 };
@@ -106,43 +123,57 @@ class AddDecomposableEvaluateWithGradient<FunctionType, true, false>
  * otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          // Check if there is at least one const Evaluate() or Gradient().
          bool HasDecomposableEvaluateGradient = traits::HasConstSignatures<
              FunctionType,
              traits::HasEvaluate,
-             traits::DecomposableEvaluateConstForm,
-             traits::DecomposableEvaluateStaticForm,
+             traits::TypedForms<MatType, GradType>::template
+                 DecomposableEvaluateConstForm,
+             traits::TypedForms<MatType, GradType>::template
+                 DecomposableEvaluateStaticForm,
              traits::HasGradient,
-             traits::DecomposableGradientConstForm,
-             traits::DecomposableGradientStaticForm>::value,
+             traits::TypedForms<MatType, GradType>::template
+                 DecomposableGradientConstForm,
+             traits::TypedForms<MatType, GradType>::template
+                 DecomposableGradientStaticForm>::value,
          bool HasDecomposableEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientConstForm>::value>
+                 traits::TypedForms<MatType, GradType>::template
+                     DecomposableEvaluateWithGradientConstForm>::value>
 class AddDecomposableEvaluateWithGradientConst
 {
  public:
   // Provide a dummy overload so the name 'EvaluateWithGradient' exists for this
   // object.
-  double EvaluateWithGradient(traits::UnconstructableType&, const size_t,
+  typename MatType::elem_type EvaluateWithGradient(
+      traits::UnconstructableType&,
+      const size_t,
       const size_t) const;
 };
 
 /**
  * Reflect the existing EvaluateWithGradient().
  */
-template<typename FunctionType, bool HasDecomposableEvaluateGradient>
-class AddDecomposableEvaluateWithGradientConst<FunctionType,
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasDecomposableEvaluateGradient>
+class AddDecomposableEvaluateWithGradientConst<FunctionType, MatType, GradType,
     HasDecomposableEvaluateGradient, true>
 {
  public:
   // Reflect the existing Evaluate().
-  double EvaluateWithGradient(const arma::mat& coordinates,
-                              const size_t begin,
-                              arma::mat& gradient,
-                              const size_t batchSize) const
+  typename MatType::elem_type EvaluateWithGradient(const MatType& coordinates,
+                                                   const size_t begin,
+                                                   GradType& gradient,
+                                                   const size_t batchSize) const
   {
     return static_cast<const FunctionType*>(
-        static_cast<const Function<FunctionType>*>(this))->EvaluateWithGradient(
+        static_cast<const Function<FunctionType,
+                                   MatType,
+                                   GradType>*>(this))->EvaluateWithGradient(
         coordinates, begin, gradient, batchSize);
   }
 };
@@ -152,8 +183,9 @@ class AddDecomposableEvaluateWithGradientConst<FunctionType,
  * Gradient() but not a decomposable const EvaluateWithGradient(), add a
  * decomposable const EvaluateWithGradient() method.
  */
-template<typename FunctionType>
-class AddDecomposableEvaluateWithGradientConst<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddDecomposableEvaluateWithGradientConst<FunctionType, MatType, GradType,
+    true, false>
 {
  public:
   /**
@@ -166,15 +198,19 @@ class AddDecomposableEvaluateWithGradientConst<FunctionType, true, false>
    * @param gradient Matrix to store the gradient into.
    * @param batchSize Number of decomposable functions to evaluate.
    */
-  double EvaluateWithGradient(const arma::mat& coordinates,
-                              const size_t begin,
-                              arma::mat& gradient,
-                              const size_t batchSize) const
+  typename MatType::elem_type EvaluateWithGradient(const MatType& coordinates,
+                                                   const size_t begin,
+                                                   GradType& gradient,
+                                                   const size_t batchSize) const
   {
-    const double objective =
-        static_cast<const Function<FunctionType>*>(this)->Evaluate(coordinates,
+    const typename MatType::elem_type objective =
+        static_cast<const Function<FunctionType,
+                                   MatType,
+                                   GradType>*>(this)->Evaluate(coordinates,
         begin, batchSize);
-    static_cast<const Function<FunctionType>*>(this)->Gradient(coordinates,
+    static_cast<const Function<FunctionType,
+                               MatType,
+                               GradType>*>(this)->Gradient(coordinates,
         begin, gradient, batchSize);
     return objective;
   }
@@ -187,36 +223,47 @@ class AddDecomposableEvaluateWithGradientConst<FunctionType, true, false>
  * nothing otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasDecomposableEvaluateGradient =
              traits::HasEvaluate<FunctionType,
-                 traits::DecomposableEvaluateStaticForm>::value &&
+                 traits::TypedForms<MatType, GradType>::template
+                     DecomposableEvaluateStaticForm>::value &&
              traits::HasGradient<FunctionType,
-                 traits::DecomposableGradientStaticForm>::value,
+                 traits::TypedForms<MatType, GradType>::template
+                     DecomposableGradientStaticForm>::value,
          bool HasDecomposableEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientStaticForm>::value>
+                 traits::TypedForms<MatType, GradType>::template
+                     DecomposableEvaluateWithGradientStaticForm>::value>
 class AddDecomposableEvaluateWithGradientStatic
 {
  public:
   // Provide a dummy overload so the name 'EvaluateWithGradient' exists for this
   // object.
-  static double EvaluateWithGradient(traits::UnconstructableType&, const size_t,
+  static typename MatType::elem_type EvaluateWithGradient(
+      traits::UnconstructableType&,
+      const size_t,
       const size_t);
 };
 
 /**
  * Reflect the existing EvaluateWithGradient().
  */
-template<typename FunctionType, bool HasDecomposableEvaluateGradient>
-class AddDecomposableEvaluateWithGradientStatic<FunctionType,
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasDecomposableEvaluateGradient>
+class AddDecomposableEvaluateWithGradientStatic<FunctionType, MatType, GradType,
     HasDecomposableEvaluateGradient, true>
 {
  public:
   // Reflect the existing Evaluate().
-  static double EvaluateWithGradient(const arma::mat& coordinates,
-                                     const size_t begin,
-                                     arma::mat& gradient,
-                                     const size_t batchSize)
+  static typename MatType::elem_type EvaluateWithGradient(
+      const MatType& coordinates,
+      const size_t begin,
+      GradType& gradient,
+      const size_t batchSize)
   {
     return FunctionType::EvaluateWithGradient(coordinates, begin, gradient,
         batchSize);
@@ -228,8 +275,9 @@ class AddDecomposableEvaluateWithGradientStatic<FunctionType,
  * Gradient() but not a decomposable static EvaluateWithGradient(), add a
  * decomposable static Gradient() method.
  */
-template<typename FunctionType>
-class AddDecomposableEvaluateWithGradientStatic<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddDecomposableEvaluateWithGradientStatic<FunctionType, MatType, GradType,
+    true, false>
 {
  public:
   /**
@@ -242,13 +290,14 @@ class AddDecomposableEvaluateWithGradientStatic<FunctionType, true, false>
    * @param gradient Matrix to store the gradient into.
    * @param batchSize Number of decomposable functions to evaluate.
    */
-  double EvaluateWithGradient(const arma::mat& coordinates,
-                              const size_t begin,
-                              arma::mat& gradient,
-                              const size_t batchSize) const
+  typename MatType::elem_type EvaluateWithGradient(
+      const MatType& coordinates,
+      const size_t begin,
+      GradType& gradient,
+      const size_t batchSize) const
   {
-    const double objective = FunctionType::Evaluate(coordinates, begin,
-        batchSize);
+    const typename MatType::elem_type objective = FunctionType::Evaluate(
+        coordinates, begin, batchSize);
     FunctionType::Gradient(coordinates, begin, gradient, batchSize);
     return objective;
   }

include/ensmallen_bits/function/add_decomposable_evaluate_with_gradient_const.hpp

@@ -1,185 +0,0 @@
-/**
- * @file add_decomposable_evaluate_with_gradient.hpp
- * @author Ryan Curtin
- *
- * Add decomposable variants of Evaluate(), Gradient(), and
- * EvaluateWithGradient().
- *
- * ensmallen is free software; you may redistribute it and/or modify it under
- * the terms of the 3-clause BSD license.  You should have received a copy of
- * the 3-clause BSD license along with ensmallen.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#ifndef ENSMALLEN_ADD_DECOMPOSABLE_EVALUATE_WITH_GRADIENT_CONST_HPP
-#define ENSMALLEN_ADD_DECOMPOSABLE_EVALUATE_WITH_GRADIENT_CONST_HPP
-
-#include "traits.hpp"
-
-namespace ens {
-
-namespace aux {
-
-template<typename FunctionType>
-using DecomposableEvaluateForm = double(FunctionType::*)(const arma::mat&,
-                                                         const size_t,
-                                                         const size_t);
-
-template<typename FunctionType>
-using DecomposableEvaluateConstForm =
-    double(FunctionType::*)(const arma::mat&, const size_t, const size_t) const;
-
-template<typename FunctionType>
-using DecomposableEvaluateStaticForm = double(*)(const arma::mat&,
-                                                 size_t,
-                                                 size_t);
-
-template<typename FunctionType>
-using DecomposableGradientForm = void(FunctionType::*)(const arma::mat&,
-                                                       const size_t,
-                                                       arma::mat&,
-                                                       const size_t);
-
-template<typename FunctionType>
-using DecomposableGradientConstForm =
-    void(FunctionType::*)(const arma::mat&,
-                          const size_t,
-                          arma::mat&,
-                          const size_t) const;
-
-template<typename FunctionType, typename... Ts>
-using DecomposableGradientStaticForm = void(*)(const arma::mat&,
-                                               const size_t,
-                                               arma::mat&);
-
-template<typename FunctionType>
-using DecomposableEvaluateWithGradientForm =
-    double(FunctionType::*)(const arma::mat&,
-                            const size_t,
-                            arma::mat&,
-                            const size_t);
-
-template<typename FunctionType>
-using DecomposableEvaluateWithGradientConstForm =
-    void(FunctionType::*)(const arma::mat&,
-                          const size_t,
-                          arma::mat&,
-                          const size_t) const;
-
-template<typename FunctionType>
-using DecomposableEvaluateWithGradientStaticForm =
-    double(*)(const arma::mat&, const size_t, arma::mat&, const size_t);
-
-} // namespace aux
-
-/**
- * The AddDecomposableEvaluateWithGradient mixin class will provide a
- * decomposable Evaluate() and Gradient() method if the given class has a
- * decomposable EvaluateWithGradient() method, or it will provide a decomposable
- * EvaluateWithGradient() method if the class has a decomposable Evaluate() and
- * Gradient() method, or it will provide nothing in any other case.
- */
-template<typename FunctionType,
-         bool HasDecomposableEvaluate =
-             aux::HasEvaluate<FunctionType,
-                 aux::DecomposableEvaluateForm>::value ||
-             aux::HasEvaluate<FunctionType,
-                 aux::DecomposableEvaluateConstForm>::value ||
-             aux::HasEvaluate<FunctionType,
-                 aux::DecomposableEvaluateStaticForm>::value,
-         bool HasGradient =
-             aux::HasGradient<FunctionType,
-                 aux::DecomposableGradientForm>::value ||
-             aux::HasGradient<FunctionType,
-                 aux::DecomposableGradientConstForm>::value ||
-             aux::HasGradient<FunctionType,
-                 aux::DecomposableGradientStaticForm>::value,
-         bool HasEvaluateWithGradient =
-             aux::HasEvaluateWithGradient<FunctionType,
-                 aux::DecomposableEvaluateWithGradientForm>::value ||
-             aux::HasEvaluateWithGradient<FunctionType,
-                 aux::DecomposableEvaluateWithGradientConstForm>::value ||
-             aux::HasEvaluateWithGradient<FunctionType,
-                 aux::DecomposableEvaluateWithGradientStaticForm>::value>
-class AddDecomposableEvaluateWithGradient : public FunctionType { };
-
-/**
- * If the FunctionType has Evaluate() and Gradient() but not
- * EvaluateWithGradient(), we will provide the latter.
- */
-template<typename FunctionType>
-class AddDecomposableEvaluateWithGradient<FunctionType, true, true, false> :
-    public FunctionType
-{
- public:
-  /**
-   * Return both the evaluated objective function and its gradient, storing the
-   * gradient in the given matrix.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param gradient Matrix to store the gradient into.
-   */
-  double EvaluateWithGradient(const arma::mat& coordinates,
-                              const size_t begin,
-                              arma::mat& gradient,
-                              const size_t batchSize)
-  {
-    const double objective = FunctionType::Evaluate(coordinates, begin,
-        batchSize);
-    FunctionType::Gradient(coordinates, begin, gradient, batchSize);
-    return objective;
-  }
-};
-
-/**
- * If the FunctionType has EvaluateWithGradient() but not Evaluate(), provide
- * that function.
- */
-template<typename FunctionType>
-class AddDecomposableEvaluateWithGradient<FunctionType, false, true, true> :
-    public FunctionType
-{
- public:
-  /**
-   * Return the objective function for the given coordinates.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   */
-  double Evaluate(const arma::mat& coordinates,
-                  const size_t begin,
-                  const size_t batchSize)
-  {
-    arma::mat gradient; // This will be ignored.
-    return FunctionType::EvaluateWithGradient(coordinates, begin, gradient,
-        batchSize);
-  }
-};
-
-/**
- * If the FunctionType has EvaluateWithGradient() but not Gradient(), provide
- * that function.
- */
-template<typename FunctionType>
-class AddDecomposableEvaluateWithGradient<FunctionType, true, false, true> :
-      public FunctionType
-{
- public:
-  /**
-   * Calculate the gradient and store it in the given matrix.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param gradient Matrix to store the gradient into.
-   */
-  void Gradient(const arma::mat& coordinates,
-                const size_t begin,
-                arma::mat& gradient,
-                const size_t batchSize)
-  {
-    // The returned objective value will be ignored.
-    (void) FunctionType::EvaluateWithGradient(coordinates, begin, gradient,
-        batchSize);
-  }
-};
-
-} // namespace ens
-
-#endif

include/ensmallen_bits/function/add_decomposable_gradient.hpp

@@ -23,12 +23,16 @@ namespace ens {
  * otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasDecomposableEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientForm>::value,
+                 traits::TypedForms<MatType, GradType>::template
+                     DecomposableEvaluateWithGradientForm>::value,
          bool HasDecomposableGradient =
              traits::HasGradient<FunctionType,
-                 traits::DecomposableGradientForm>::value>
+                 traits::TypedForms<MatType, GradType>::template
+                     DecomposableGradientForm>::value>
 class AddDecomposableGradient
 {
  public:
@@ -39,20 +43,25 @@ class AddDecomposableGradient
 /**
  * Reflect the existing Gradient().
  */
-template<typename FunctionType, bool HasDecomposableEvaluateWithGradient>
-class AddDecomposableGradient<FunctionType, HasDecomposableEvaluateWithGradient,
-    true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasDecomposableEvaluateWithGradient>
+class AddDecomposableGradient<FunctionType, MatType, GradType,
+    HasDecomposableEvaluateWithGradient, true>
 {
  public:
   // Reflect the existing Gradient().
-  void Gradient(const arma::mat& coordinates,
+  void Gradient(const MatType& coordinates,
                 const size_t begin,
-                arma::mat& gradient,
+                GradType& gradient,
                 const size_t batchSize)
   {
     static_cast<FunctionType*>(
-        static_cast<Function<FunctionType>*>(this))->Gradient(coordinates,
-        begin, gradient, batchSize);
+        static_cast<Function<FunctionType,
+                             MatType,
+                             GradType>*>(this))->Gradient(
+        coordinates, begin, gradient, batchSize);
   }
 };
 
@@ -60,8 +69,8 @@ class AddDecomposableGradient<FunctionType, HasDecomposableEvaluateWithGradient,
  * If we have a decomposable EvaluateWithGradient() but not a decomposable
  * Gradient(), add a decomposable Evaluate() method.
  */
-template<typename FunctionType>
-class AddDecomposableGradient<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddDecomposableGradient<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -73,13 +82,15 @@ class AddDecomposableGradient<FunctionType, true, false>
    * @param gradient Matrix to store the gradient into.
    * @param batchSize Number of decomposable functions to calculate for.
    */
-  void Gradient(const arma::mat& coordinates,
+  void Gradient(const MatType& coordinates,
                 const size_t begin,
-                arma::mat& gradient,
+                GradType& gradient,
                 const size_t batchSize)
   {
     // The returned objective value will be ignored.
-    (void) static_cast<Function<FunctionType>*>(this)->EvaluateWithGradient(
+    (void) static_cast<Function<FunctionType,
+                                MatType,
+                                GradType>*>(this)->EvaluateWithGradient(
         coordinates, begin, gradient, batchSize);
   }
 };
@@ -90,12 +101,16 @@ class AddDecomposableGradient<FunctionType, true, false>
  * exists, or nothing otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasDecomposableEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientConstForm>::value,
+                 traits::TypedForms<MatType, GradType>::template
+                     DecomposableEvaluateWithGradientConstForm>::value,
          bool HasDecomposableGradient =
              traits::HasGradient<FunctionType,
-                 traits::DecomposableGradientConstForm>::value>
+                 traits::TypedForms<MatType, GradType>::template
+                     DecomposableGradientConstForm>::value>
 class AddDecomposableGradientConst
 {
  public:
@@ -106,19 +121,24 @@ class AddDecomposableGradientConst
 /**
  * Reflect the existing Gradient().
  */
-template<typename FunctionType, bool HasDecomposableEvaluateWithGradient>
-class AddDecomposableGradientConst<FunctionType,
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasDecomposableEvaluateWithGradient>
+class AddDecomposableGradientConst<FunctionType, MatType, GradType,
     HasDecomposableEvaluateWithGradient, true>
 {
  public:
   // Reflect the existing Gradient().
-  void Gradient(const arma::mat& coordinates,
+  void Gradient(const MatType& coordinates,
                 const size_t begin,
-                arma::mat& gradient,
+                GradType& gradient,
                 const size_t batchSize) const
   {
     static_cast<const FunctionType*>(
-        static_cast<const Function<FunctionType>*>(this))->Gradient(coordinates,
+        static_cast<const Function<FunctionType,
+                                   MatType,
+                                   GradType>*>(this))->Gradient(coordinates,
         begin, gradient, batchSize);
   }
 };
@@ -127,8 +147,8 @@ class AddDecomposableGradientConst<FunctionType,
  * If we have a decomposable const EvaluateWithGradient() but not a decomposable
  * const Gradient(), add a decomposable const Gradient() method.
  */
-template<typename FunctionType>
-class AddDecomposableGradientConst<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddDecomposableGradientConst<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -140,15 +160,17 @@ class AddDecomposableGradientConst<FunctionType, true, false>
    * @param gradient Matrix to store the gradient into.
    * @param batchSize Number of decomposable functions to calculate for.
    */
-  void Gradient(const arma::mat& coordinates,
+  void Gradient(const MatType& coordinates,
                 const size_t begin,
-                arma::mat& gradient,
+                GradType& gradient,
                 const size_t batchSize) const
   {
     // The returned objective value will be ignored.
     (void) static_cast<
-        const Function<FunctionType>*>(this)->EvaluateWithGradient(coordinates,
-        begin, gradient, batchSize);
+        const Function<FunctionType,
+                       MatType,
+                       GradType>*>(this)->EvaluateWithGradient(
+        coordinates, begin, gradient, batchSize);
   }
 };
 
@@ -158,12 +180,16 @@ class AddDecomposableGradientConst<FunctionType, true, false>
  * exists, or nothing otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasDecomposableEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientStaticForm>::value,
+                 traits::TypedForms<MatType, GradType>::template
+                     DecomposableEvaluateWithGradientStaticForm>::value,
          bool HasDecomposableGradient =
              traits::HasGradient<FunctionType,
-                 traits::DecomposableGradientStaticForm>::value>
+                 traits::TypedForms<MatType, GradType>::template
+                     DecomposableGradientStaticForm>::value>
 class AddDecomposableGradientStatic
 {
  public:
@@ -176,15 +202,18 @@ class AddDecomposableGradientStatic
 /**
  * Reflect the existing Gradient().
  */
-template<typename FunctionType, bool HasDecomposableEvaluateWithGradient>
-class AddDecomposableGradientStatic<FunctionType,
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasDecomposableEvaluateWithGradient>
+class AddDecomposableGradientStatic<FunctionType, MatType, GradType,
     HasDecomposableEvaluateWithGradient, true>
 {
  public:
   // Reflect the existing Gradient().
-  static void Gradient(const arma::mat& coordinates,
+  static void Gradient(const MatType& coordinates,
                        const size_t begin,
-                       arma::mat& gradient,
+                       GradType& gradient,
                        const size_t batchSize)
   {
     FunctionType::Gradient(coordinates, begin, gradient, batchSize);
@@ -195,8 +224,9 @@ class AddDecomposableGradientStatic<FunctionType,
  * If we have a decomposable EvaluateWithGradient() but not a decomposable
  * Gradient(), add a decomposable Gradient() method.
  */
-template<typename FunctionType>
-class AddDecomposableGradientStatic<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddDecomposableGradientStatic<FunctionType, MatType, GradType, true,
+    false>
 {
  public:
   /**
@@ -208,9 +238,9 @@ class AddDecomposableGradientStatic<FunctionType, true, false>
    * @param gradient Matrix to store the gradient into.
    * @param batchSize Number of decomposable functions to calculate for.
    */
-  static void Gradient(const arma::mat& coordinates,
+  static void Gradient(const MatType& coordinates,
                        const size_t begin,
-                       arma::mat& gradient,
+                       GradType& gradient,
                        const size_t batchSize)
   {
     // The returned objective value will be ignored.

include/ensmallen_bits/function/add_evaluate.hpp

@@ -22,30 +22,44 @@ namespace ens {
  * FunctionType has EvaluateWithGradient(), or nothing otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::EvaluateWithGradientForm>::value,
+                 traits::TypedForms<MatType, GradType>::template
+                     EvaluateWithGradientForm
+              >::value,
          bool HasEvaluate =
-             traits::HasEvaluate<FunctionType, traits::EvaluateForm>::value>
+             traits::HasEvaluate<FunctionType,
+                  traits::TypedForms<MatType, GradType>::template
+                     EvaluateForm>::value>
 class AddEvaluate
 {
  public:
   // Provide a dummy overload so the name 'Evaluate' exists for this object.
-  double Evaluate(traits::UnconstructableType&);
+  typename MatType::elem_type Evaluate(traits::UnconstructableType&);
 };
 
 /**
  * Reflect the existing Evaluate().
  */
-template<typename FunctionType, bool HasEvaluateWithGradient>
-class AddEvaluate<FunctionType, HasEvaluateWithGradient, true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasEvaluateWithGradient>
+class AddEvaluate<FunctionType,
+                  MatType,
+                  GradType,
+                  HasEvaluateWithGradient,
+                  true>
 {
  public:
   // Reflect the existing Evaluate().
-  double Evaluate(const arma::mat& coordinates)
+  typename MatType::elem_type Evaluate(const MatType& coordinates)
   {
-    return static_cast<FunctionType*>(static_cast<Function<FunctionType>*>(
-        this))->Evaluate(coordinates);
+    return static_cast<FunctionType*>(
+        static_cast<Function<FunctionType,
+                             MatType, GradType>*>(this))->Evaluate(coordinates);
   }
 };
 
@@ -53,8 +67,8 @@ class AddEvaluate<FunctionType, HasEvaluateWithGradient, true>
  * If we have EvaluateWithGradient() but no existing Evaluate(), add an
  * Evaluate() method.
  */
-template<typename FunctionType>
-class AddEvaluate<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddEvaluate<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -62,10 +76,12 @@ class AddEvaluate<FunctionType, true, false>
    *
    * @param coordinates Coordinates to evaluate the function at.
    */
-  double Evaluate(const arma::mat& coordinates)
+  typename MatType::elem_type Evaluate(const MatType& coordinates)
   {
-    arma::mat gradient; // This will be ignored.
-    return static_cast<Function<FunctionType>*>(this)->EvaluateWithGradient(
+    GradType gradient; // This will be ignored.
+    return static_cast<Function<FunctionType,
+                                MatType,
+                                GradType>*>(this)->EvaluateWithGradient(
         coordinates, gradient);
   }
 };
@@ -76,32 +92,47 @@ class AddEvaluate<FunctionType, true, false>
  * otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::EvaluateWithGradientConstForm>::value,
+                 traits::TypedForms<MatType,
+                                    GradType>::template
+                     EvaluateWithGradientConstForm
+             >::value,
          bool HasEvaluate =
              traits::HasEvaluate<FunctionType,
-                 traits::EvaluateConstForm>::value>
+                 traits::TypedForms<MatType, GradType>::template
+                     EvaluateConstForm
+             >::value>
 class AddEvaluateConst
 {
  public:
   // Provide a dummy overload so the name 'Evaluate' exists for this object.
-  double Evaluate(traits::UnconstructableType&) const;
+  typename MatType::elem_type Evaluate(traits::UnconstructableType&) const;
 };
 
 /**
  * Reflect the existing Evaluate().
  */
-template<typename FunctionType, bool HasEvaluateWithGradient>
-class AddEvaluateConst<FunctionType, HasEvaluateWithGradient, true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasEvaluateWithGradient>
+class AddEvaluateConst<FunctionType,
+                       MatType,
+                       GradType,
+                       HasEvaluateWithGradient,
+                       true>
 {
  public:
   // Reflect the existing Evaluate().
-  double Evaluate(const arma::mat& coordinates) const
+  typename MatType::elem_type Evaluate(const MatType& coordinates) const
   {
-    return static_cast<const FunctionType*>(static_cast<const
-Function<FunctionType>*>(this))->Evaluate(
-        coordinates);
+    return static_cast<const FunctionType*>(
+        static_cast<const Function<FunctionType,
+                                   MatType,
+                                   GradType>*>(this))->Evaluate(coordinates);
   }
 };
 
@@ -109,8 +140,8 @@ Function<FunctionType>*>(this))->Evaluate(
  * If we have EvaluateWithGradient() but no existing Evaluate(), add an
  * Evaluate() without a using directive to make the base Evaluate() accessible.
  */
-template<typename FunctionType>
-class AddEvaluateConst<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddEvaluateConst<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -118,12 +149,14 @@ class AddEvaluateConst<FunctionType, true, false>
    *
    * @param coordinates Coordinates to evaluate the function at.
    */
-  double Evaluate(const arma::mat& coordinates) const
+  typename MatType::elem_type Evaluate(const MatType& coordinates) const
   {
-    arma::mat gradient; // This will be ignored.
+    GradType gradient; // This will be ignored.
     return static_cast<
-        const Function<FunctionType>*>(this)->EvaluateWithGradient(coordinates,
-        gradient);
+        const Function<FunctionType,
+                       MatType,
+                       GradType>*>(this)->EvaluateWithGradient(coordinates,
+                                                               gradient);
   }
 };
 
@@ -133,28 +166,43 @@ class AddEvaluateConst<FunctionType, true, false>
  * otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::EvaluateWithGradientStaticForm>::value,
+                 traits::TypedForms<MatType,
+                                    GradType>::template
+                 EvaluateWithGradientStaticForm
+             >::value,
          bool HasEvaluate =
              traits::HasEvaluate<FunctionType,
-                 traits::EvaluateStaticForm>::value>
+                 traits::TypedForms<MatType, GradType>::template
+                 EvaluateStaticForm
+             >::value>
 class AddEvaluateStatic
 {
  public:
   // Provide a dummy overload so the name 'Evaluate' exists for this object.
-  static double Evaluate(traits::UnconstructableType&);
+  static typename MatType::elem_type Evaluate(traits::UnconstructableType&);
 };
 
 /**
  * Reflect the existing Evaluate().
  */
-template<typename FunctionType, bool HasEvaluateWithGradient>
-class AddEvaluateStatic<FunctionType, HasEvaluateWithGradient, true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasEvaluateWithGradient>
+class AddEvaluateStatic<FunctionType,
+                        MatType,
+                        GradType,
+                        HasEvaluateWithGradient,
+                        true>
 {
  public:
   // Reflect the existing Evaluate().
-  static double Evaluate(const arma::mat& coordinates)
+  static typename MatType::elem_type Evaluate(
+      const MatType& coordinates)
   {
     return FunctionType::Evaluate(coordinates);
   }
@@ -164,8 +212,8 @@ class AddEvaluateStatic<FunctionType, HasEvaluateWithGradient, true>
  * If we have EvaluateWithGradient() but no existing Evaluate(), add an
  * Evaluate() without a using directive to make the base Evaluate() accessible.
  */
-template<typename FunctionType>
-class AddEvaluateStatic<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddEvaluateStatic<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -173,9 +221,9 @@ class AddEvaluateStatic<FunctionType, true, false>
    *
    * @param coordinates Coordinates to evaluate the function at.
    */
-  static double Evaluate(const arma::mat& coordinates)
+  static typename MatType::elem_type Evaluate(const MatType& coordinates)
   {
-    arma::mat gradient; // This will be ignored.
+    GradType gradient; // This will be ignored.
     return FunctionType::EvaluateWithGradient(coordinates, gradient);
   }
 };

include/ensmallen_bits/function/add_evaluate_with_gradient.hpp

@@ -24,40 +24,55 @@ namespace ens {
  * and Gradient(), or it will provide nothing otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          // Check if there is at least one non-const Evaluate() or Gradient().
          bool HasEvaluateGradient = traits::HasNonConstSignatures<
              FunctionType,
              traits::HasEvaluate,
-             traits::EvaluateForm,
-             traits::EvaluateConstForm,
-             traits::EvaluateStaticForm,
+             traits::TypedForms<MatType, GradType>::template EvaluateForm,
+             traits::TypedForms<MatType, GradType>::template EvaluateConstForm,
+             traits::TypedForms<MatType, GradType>::template EvaluateStaticForm,
              traits::HasGradient,
-             traits::GradientForm,
-             traits::GradientConstForm,
-             traits::GradientStaticForm>::value,
+             traits::TypedForms<MatType, GradType>::template GradientForm,
+             traits::TypedForms<MatType, GradType>::template GradientConstForm,
+             traits::TypedForms<MatType, GradType>::template GradientStaticForm
+         >::value,
          bool HasEvaluateWithGradient = traits::HasEvaluateWithGradient<
              FunctionType,
-             traits::EvaluateWithGradientForm>::value>
+             traits::TypedForms<MatType, GradType>::template
+                 EvaluateWithGradientForm>::value>
 class AddEvaluateWithGradient
 {
  public:
   // Provide a dummy overload so the name 'EvaluateWithGradient' exists for this
   // object.
-  double EvaluateWithGradient(traits::UnconstructableType&);
+  typename MatType::elem_type EvaluateWithGradient(
+      traits::UnconstructableType&);
 };
 
 /**
  * Reflect the existing EvaluateWithGradient().
  */
-template<typename FunctionType, bool HasEvaluateGradient>
-class AddEvaluateWithGradient<FunctionType, HasEvaluateGradient, true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasEvaluateGradient>
+class AddEvaluateWithGradient<FunctionType,
+                              MatType,
+                              GradType,
+                              HasEvaluateGradient,
+                              true>
 {
  public:
   // Reflect the existing EvaluateWithGradient().
-  double EvaluateWithGradient(const arma::mat& coordinates, arma::mat& gradient)
+  typename MatType::elem_type EvaluateWithGradient(
+      const MatType& coordinates, GradType& gradient)
   {
     return static_cast<FunctionType*>(
-        static_cast<Function<FunctionType>*>(this))->EvaluateWithGradient(
+        static_cast<Function<FunctionType,
+                             MatType,
+                             GradType>*>(this))->EvaluateWithGradient(
         coordinates, gradient);
   }
 };
@@ -66,8 +81,8 @@ class AddEvaluateWithGradient<FunctionType, HasEvaluateGradient, true>
  * If the FunctionType has Evaluate() and Gradient(), provide
  * EvaluateWithGradient().
  */
-template<typename FunctionType>
-class AddEvaluateWithGradient<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddEvaluateWithGradient<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -77,12 +92,15 @@ class AddEvaluateWithGradient<FunctionType, true, false>
    * @param coordinates Coordinates to evaluate the function at.
    * @param gradient Matrix to store the gradient into.
    */
-  double EvaluateWithGradient(const arma::mat& coordinates,
-                              arma::mat& gradient)
+  typename MatType::elem_type EvaluateWithGradient(const MatType& coordinates,
+                                                   GradType& gradient)
   {
-    const double objective =
-        static_cast<Function<FunctionType>*>(this)->Evaluate(coordinates);
-    static_cast<Function<FunctionType>*>(this)->Gradient(coordinates, gradient);
+    const typename MatType::elem_type objective =
+        static_cast<Function<FunctionType,
+                             MatType, GradType>*>(this)->Evaluate(coordinates);
+    static_cast<Function<FunctionType,
+                         MatType,
+                         GradType>*>(this)->Gradient(coordinates, gradient);
     return objective;
   }
 };
@@ -93,39 +111,54 @@ class AddEvaluateWithGradient<FunctionType, true, false>
  * Evaluate() const and Gradient() const, or it will provide nothing otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          // Check if there is at least one const Evaluate() or Gradient().
          bool HasEvaluateGradient = traits::HasConstSignatures<
              FunctionType,
              traits::HasEvaluate,
-             traits::EvaluateConstForm,
-             traits::EvaluateStaticForm,
+             traits::TypedForms<MatType, GradType>::template EvaluateConstForm,
+             traits::TypedForms<MatType, GradType>::template EvaluateStaticForm,
              traits::HasGradient,
-             traits::GradientConstForm,
-             traits::GradientStaticForm>::value,
+             traits::TypedForms<MatType, GradType>::template GradientConstForm,
+             traits::TypedForms<MatType, GradType>::template GradientStaticForm
+         >::value,
          bool HasEvaluateWithGradient = traits::HasEvaluateWithGradient<
              FunctionType,
-             traits::EvaluateWithGradientConstForm>::value>
+             traits::TypedForms<
+                 MatType, GradType
+             >::template EvaluateWithGradientConstForm>::value>
 class AddEvaluateWithGradientConst
 {
  public:
   // Provide a dummy overload so the name 'EvaluateWithGradient' exists for this
   // object.
-  double EvaluateWithGradient(traits::UnconstructableType&) const;
+  typename MatType::elem_type EvaluateWithGradient(
+      traits::UnconstructableType&) const;
 };
 
 /**
  * Reflect the existing EvaluateWithGradient().
  */
-template<typename FunctionType, bool HasEvaluateGradient>
-class AddEvaluateWithGradientConst<FunctionType, HasEvaluateGradient, true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasEvaluateGradient>
+class AddEvaluateWithGradientConst<FunctionType,
+                                   MatType,
+                                   GradType,
+                                   HasEvaluateGradient,
+                                   true>
 {
  public:
   // Reflect the existing EvaluateWithGradient().
-  double EvaluateWithGradient(const arma::mat& coordinates, arma::mat& gradient)
-      const
+  typename MatType::elem_type EvaluateWithGradient(
+      const MatType& coordinates, GradType& gradient) const
   {
     return static_cast<const FunctionType*>(
-        static_cast<const Function<FunctionType>*>(this))->EvaluateWithGradient(
+        static_cast<const Function<FunctionType,
+                                   MatType,
+                                   GradType>*>(this))->EvaluateWithGradient(
         coordinates, gradient);
   }
 };
@@ -134,8 +167,8 @@ class AddEvaluateWithGradientConst<FunctionType, HasEvaluateGradient, true>
  * If the FunctionType has Evaluate() const and Gradient() const, provide
  * EvaluateWithGradient() const.
  */
-template<typename FunctionType>
-class AddEvaluateWithGradientConst<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddEvaluateWithGradientConst<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -145,13 +178,17 @@ class AddEvaluateWithGradientConst<FunctionType, true, false>
    * @param coordinates Coordinates to evaluate the function at.
    * @param gradient Matrix to store the gradient into.
    */
-  double EvaluateWithGradient(const arma::mat& coordinates,
-                              arma::mat& gradient) const
+  typename MatType::elem_type EvaluateWithGradient(const MatType& coordinates,
+                                                   GradType& gradient) const
   {
-    const double objective =
-        static_cast<const Function<FunctionType>*>(this)->Evaluate(coordinates);
-    static_cast<const Function<FunctionType>*>(this)->Gradient(coordinates,
-        gradient);
+    const typename MatType::elem_type objective =
+        static_cast<const Function<FunctionType,
+                                   MatType,
+                                   GradType>*>(this)->Evaluate(coordinates);
+    static_cast<const Function<FunctionType,
+                               MatType,
+                               GradType>*>(this)->Gradient(coordinates,
+                                                           gradient);
     return objective;
   }
 };
@@ -163,32 +200,49 @@ class AddEvaluateWithGradientConst<FunctionType, true, false>
  * otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasEvaluateGradient =
              traits::HasEvaluate<FunctionType,
-                 traits::EvaluateStaticForm>::value &&
+                 traits::TypedForms<MatType, GradType>::template
+                     EvaluateStaticForm
+             >::value &&
              traits::HasGradient<FunctionType,
-                 traits::GradientStaticForm>::value,
+                 traits::TypedForms<MatType, GradType>::template
+                     GradientStaticForm
+             >::value,
          bool HasEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::EvaluateWithGradientStaticForm>::value>
+                 traits::TypedForms<MatType,
+                                    GradType>::template
+                     EvaluateWithGradientStaticForm
+             >::value>
 class AddEvaluateWithGradientStatic
 {
  public:
   // Provide a dummy overload so the name 'EvaluateWithGradient' exists for this
   // object.
-  static double EvaluateWithGradient(traits::UnconstructableType&);
+  static typename MatType::elem_type EvaluateWithGradient(
+      traits::UnconstructableType&);
 };
 
 /**
  * Reflect the existing EvaluateWithGradient().
  */
-template<typename FunctionType, bool HasEvaluateGradient>
-class AddEvaluateWithGradientStatic<FunctionType, HasEvaluateGradient, true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasEvaluateGradient>
+class AddEvaluateWithGradientStatic<FunctionType,
+                                    MatType,
+                                    GradType,
+                                    HasEvaluateGradient,
+                                    true>
 {
  public:
   // Reflect the existing EvaluateWithGradient().
-  static double EvaluateWithGradient(const arma::mat& coordinates,
-                                     arma::mat& gradient)
+  static typename MatType::elem_type EvaluateWithGradient(
+      const MatType& coordinates, GradType& gradient)
   {
     return FunctionType::EvaluateWithGradient(coordinates, gradient);
   }
@@ -198,8 +252,12 @@ class AddEvaluateWithGradientStatic<FunctionType, HasEvaluateGradient, true>
  * If the FunctionType has static Evaluate() and static Gradient(), provide
  * static EvaluateWithGradient().
  */
-template<typename FunctionType>
-class AddEvaluateWithGradientStatic<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddEvaluateWithGradientStatic<FunctionType,
+                                    MatType,
+                                    GradType,
+                                    true,
+                                    false>
 {
  public:
   /**
@@ -209,10 +267,11 @@ class AddEvaluateWithGradientStatic<FunctionType, true, false>
    * @param coordinates Coordinates to evaluate the function at.
    * @param gradient Matrix to store the gradient into.
    */
-  static double EvaluateWithGradient(const arma::mat& coordinates,
-                                     arma::mat& gradient)
+  static typename MatType::elem_type EvaluateWithGradient(
+      const MatType& coordinates, GradType& gradient)
   {
-    const double objective = FunctionType::Evaluate(coordinates);
+    const typename MatType::elem_type objective =
+        FunctionType::Evaluate(coordinates);
     FunctionType::Gradient(coordinates, gradient);
     return objective;
   }

include/ensmallen_bits/function/add_gradient.hpp

@@ -22,11 +22,16 @@ namespace ens {
  * FunctionType has EvaluateWithGradient(), or nothing otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::EvaluateWithGradientForm>::value,
+                 traits::TypedForms<MatType, GradType>::template
+                     EvaluateWithGradientForm
+             >::value,
          bool HasGradient = traits::HasGradient<FunctionType,
-             traits::GradientForm>::value>
+             traits::TypedForms<MatType, GradType>::template 
+                     GradientForm>::value>
 class AddGradient
 {
  public:
@@ -37,15 +42,25 @@ class AddGradient
 /**
  * Reflect the existing Gradient().
  */
-template<typename FunctionType, bool HasEvaluateWithGradient>
-class AddGradient<FunctionType, HasEvaluateWithGradient, true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasEvaluateWithGradient>
+class AddGradient<FunctionType,
+                  MatType,
+                  GradType,
+                  HasEvaluateWithGradient,
+                  true>
 {
  public:
   // Reflect the existing Gradient().
-  void Gradient(const arma::mat& coordinates, arma::mat& gradient)
+  void Gradient(const MatType& coordinates, GradType& gradient)
   {
-    static_cast<FunctionType*>(static_cast<Function<FunctionType>*>(
-        this))->Gradient(coordinates, gradient);
+    static_cast<FunctionType*>(
+        static_cast<Function<FunctionType,
+                             MatType,
+                             GradType>*>(this))->Gradient(coordinates,
+                                                          gradient);
   }
 };
 
@@ -53,8 +68,8 @@ class AddGradient<FunctionType, HasEvaluateWithGradient, true>
  * If we have EvaluateWithGradient() but no existing Gradient(), add an
  * Gradient() without a using directive to make the base Gradient() accessible.
  */
-template<typename FunctionType>
-class AddGradient<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddGradient<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -63,10 +78,12 @@ class AddGradient<FunctionType, true, false>
    * @param coordinates Coordinates to evaluate the function at.
    * @param gradient Matrix to store the gradient into.
    */
-  void Gradient(const arma::mat& coordinates, arma::mat& gradient)
+  void Gradient(const MatType& coordinates, GradType& gradient)
   {
     // The returned objective value will be ignored.
-    (void) static_cast<Function<FunctionType>*>(this)->EvaluateWithGradient(
+    (void) static_cast<Function<FunctionType,
+                                MatType,
+                                GradType>*>(this)->EvaluateWithGradient(
         coordinates, gradient);
   }
 };
@@ -76,11 +93,17 @@ class AddGradient<FunctionType, true, false>
  * given FunctionType has EvaluateWithGradient() const, or nothing otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::EvaluateWithGradientConstForm>::value,
+                 traits::TypedForms<MatType,
+                                    GradType>::template
+                     EvaluateWithGradientConstForm
+             >::value,
          bool HasGradient = traits::HasGradient<FunctionType,
-             traits::GradientConstForm>::value>
+             traits::TypedForms<MatType, GradType>::template GradientConstForm
+         >::value>
 class AddGradientConst
 {
  public:
@@ -91,16 +114,25 @@ class AddGradientConst
 /**
  * Reflect the existing Gradient().
  */
-template<typename FunctionType, bool HasEvaluateWithGradient>
-class AddGradientConst<FunctionType, HasEvaluateWithGradient, true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasEvaluateWithGradient>
+class AddGradientConst<FunctionType,
+                       MatType,
+                       GradType,
+                       HasEvaluateWithGradient,
+                       true>
 {
  public:
   // Reflect the existing Gradient().
-  void Gradient(const arma::mat& coordinates, arma::mat& gradient) const
+  void Gradient(const MatType& coordinates, GradType& gradient) const
   {
-    static_cast<const FunctionType*>(static_cast<const
-Function<FunctionType>*>(this))->Gradient(coordinates,
-        gradient);
+    static_cast<const FunctionType*>(
+        static_cast<const Function<FunctionType,
+                                   MatType,
+                                   GradType>*>(this))->Gradient(coordinates,
+                                                                gradient);
   }
 };
 
@@ -108,8 +140,8 @@ Function<FunctionType>*>(this))->Gradient(coordinates,
  * If we have EvaluateWithGradient() but no existing Gradient(), add a
  * Gradient() without a using directive to make the base Gradient() accessible.
  */
-template<typename FunctionType>
-class AddGradientConst<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddGradientConst<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -118,12 +150,14 @@ class AddGradientConst<FunctionType, true, false>
    * @param coordinates Coordinates to evaluate the function at.
    * @param gradient Matrix to store the gradient into.
    */
-  void Gradient(const arma::mat& coordinates, arma::mat& gradient) const
+  void Gradient(const MatType& coordinates, GradType& gradient) const
   {
     // The returned objective value will be ignored.
     (void) static_cast<
-        const Function<FunctionType>*>(this)->EvaluateWithGradient(coordinates,
-        gradient);
+        const Function<FunctionType,
+                       MatType,
+                       GradType>*>(this)->EvaluateWithGradient(coordinates,
+                                                               gradient);
   }
 };
 
@@ -132,11 +166,17 @@ class AddGradientConst<FunctionType, true, false>
  * given FunctionType has static EvaluateWithGradient(), or nothing otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::EvaluateWithGradientStaticForm>::value,
+                 traits::TypedForms<MatType,
+                                    GradType>::template
+                     EvaluateWithGradientStaticForm
+             >::value,
          bool HasGradient = traits::HasGradient<FunctionType,
-             traits::GradientStaticForm>::value>
+             traits::TypedForms<MatType, GradType>::template GradientStaticForm
+         >::value>
 class AddGradientStatic
 {
  public:
@@ -147,12 +187,19 @@ class AddGradientStatic
 /**
  * Reflect the existing Gradient().
  */
-template<typename FunctionType, bool HasEvaluateWithGradient>
-class AddGradientStatic<FunctionType, HasEvaluateWithGradient, true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasEvaluateWithGradient>
+class AddGradientStatic<FunctionType,
+                        MatType,
+                        GradType,
+                        HasEvaluateWithGradient,
+                        true>
 {
  public:
   // Reflect the existing Gradient().
-  static void Gradient(const arma::mat& coordinates, arma::mat& gradient)
+  static void Gradient(const MatType& coordinates, GradType& gradient)
   {
     FunctionType::Gradient(coordinates, gradient);
   }
@@ -162,8 +209,8 @@ class AddGradientStatic<FunctionType, HasEvaluateWithGradient, true>
  * If we have EvaluateWithGradient() but no existing Gradient(), add a
  * Gradient() without a using directive to make the base Gradient() accessible.
  */
-template<typename FunctionType>
-class AddGradientStatic<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddGradientStatic<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -172,7 +219,7 @@ class AddGradientStatic<FunctionType, true, false>
    * @param coordinates Coordinates to evaluate the function at.
    * @param gradient Matrix to store the gradient into.
    */
-  static void Gradient(const arma::mat& coordinates, arma::mat& gradient)
+  static void Gradient(const MatType& coordinates, GradType& gradient)
   {
     // The returned objective value will be ignored.
     (void) FunctionType::EvaluateWithGradient(coordinates, gradient);

include/ensmallen_bits/function/arma_traits.hpp

@@ -0,0 +1,151 @@
+/**
+ * @file arma_traits.hpp
+ * @author Ryan Curtin
+ *
+ * Given an Armadillo type, determine its "true" base type.
+ */
+#ifndef ENSMALLEN_FUNCTION_ARMA_TRAITS_HPP
+#define ENSMALLEN_FUNCTION_ARMA_TRAITS_HPP
+
+namespace ens {
+
+/**
+ * Extract the base type of a matrix (i.e. if it is a column, return the matrix
+ * type).  If the type is unknown (or not a derived type) we just return the
+ * type itself as the typedef BaseMatType.
+ */
+
+template<typename MatType>
+struct MatTypeTraits
+{
+  typedef MatType BaseMatType;
+};
+
+template<typename eT>
+struct MatTypeTraits<arma::Col<eT>>
+{
+  typedef arma::Mat<eT> BaseMatType;
+};
+
+template<typename eT>
+struct MatTypeTraits<arma::Row<eT>>
+{
+  typedef arma::Row<eT> BaseMatType;
+};
+
+template<typename eT>
+struct MatTypeTraits<arma::SpCol<eT>>
+{
+  typedef arma::SpMat<eT> BaseMatType;
+};
+
+template<typename eT>
+struct MatTypeTraits<arma::SpRow<eT>>
+{
+  typedef arma::SpMat<eT> BaseMatType;
+};
+
+/**
+ * Disable usage of arma::subviews and related types for optimizers.  It might
+ * be nice to also explicitly disable Armadillo expressions, but we'll hope for
+ * now nobody even tries that, since those aren't even lvalues and thus can't
+ * really work.
+ */
+
+template<typename eT>
+struct MatTypeTraits<arma::subview<eT>>
+{
+  static_assert(sizeof(arma::subview<eT>) == 0,
+      "Armadillo subviews cannot be passed to Optimize()!  Create a matrix "
+      "or a matrix alias instead!");
+};
+
+template<typename eT>
+struct MatTypeTraits<arma::subview_col<eT>>
+{
+  static_assert(sizeof(arma::subview_col<eT>) == 0,
+      "Armadillo subviews cannot be passed to Optimize()!  Create a matrix "
+      "or a matrix alias instead!");
+};
+
+template<typename eT>
+struct MatTypeTraits<arma::SpSubview<eT>>
+{
+  static_assert(sizeof(arma::SpSubview<eT>) == 0,
+      "Armadillo subviews cannot be passed to Optimize()!  Create a matrix "
+      "or a matrix alias instead!");
+};
+
+template<typename eT>
+struct MatTypeTraits<arma::Cube<eT>>
+{
+  static_assert(sizeof(arma::Cube<eT>) == 0,
+      "Armadillo cubes cannot be passed to Optimize()!  Create a matrix "
+      "or a matrix alias instead!");
+};
+
+/**
+ * Issue a fatal error if the type is not an Armadillo double or floating point
+ * sparse or dense matrix.
+ */
+
+template<typename MatType>
+void RequireDenseFloatingPointType()
+{
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(sizeof(MatType) == 0,
+      "The given MatType must be arma::mat or arma::fmat or it is not known "
+      "to work!  If you would like to try anyway, set the preprocessor macro "
+      "ENS_DISABLE_TYPE_CHECKS before including ensmallen.hpp.  However, you "
+      "get to pick up all the pieces if there is a failure!");
+#endif
+}
+
+template<>
+inline void RequireDenseFloatingPointType<arma::mat>() { }
+template<>
+inline void RequireDenseFloatingPointType<arma::fmat>() { }
+
+template<typename MatType>
+void RequireFloatingPointType()
+{
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(sizeof(MatType) == 0,
+      "The given MatType must be arma::mat, arma::fmat, arma::sp_mat, or "
+      "arma::sp_fmat, or it is not known to work!  If you would like to try "
+      "anyway, set the preprocessor macro ENS_DISABLE_TYPE_CHECKS before "
+      "including ensmallen.hpp.  However, you get to pick up all the pieces if "
+      "there is a failure!");
+#endif
+}
+
+template<>
+inline void RequireFloatingPointType<arma::mat>() { }
+template<>
+inline void RequireFloatingPointType<arma::fmat>() { }
+template<>
+inline void RequireFloatingPointType<arma::sp_mat>() { }
+template<>
+inline void RequireFloatingPointType<arma::sp_fmat>() { }
+
+/**
+ * Require that the internal element type of the matrix type and gradient type
+ * are the same.  A static_assert() will fail if not.
+ */
+template<typename MatType, typename GradType>
+void RequireSameInternalTypes()
+{
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(std::is_same<typename MatType::elem_type,
+                             typename GradType::elem_type>::value,
+      "The internal element types of the given MatType and GradType must be "
+      "identical, or it is not known to work!  If you would like to try "
+      "anyway, set the preprocessor macro ENS_DISABLE_TYPE_CHECKS before "
+      "including ensmallen.hpp.  However, you get to pick up all the pieces if "
+      "there is a failure!");
+#endif
+}
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/function/sfinae_utility.hpp

@@ -105,6 +105,27 @@ struct MethodFormDetector<Class, MethodForm, 7>
 //! Utility struct for checking signatures.
 template<typename U, U> struct SigCheck : std::true_type {};
 
+template<typename... Args>
+struct pack {};
+
+template<typename Func>
+struct FunctionTypes {};
+
+template<typename R, typename... A>
+struct FunctionTypes<R(A...)>
+{
+  typedef R Ret;
+  using Args = pack<A...>;
+};
+
+template<typename R, typename C, typename... A>
+struct FunctionTypes<R(C::*)(A...)>
+{
+  typedef R Ret;
+  typedef C Class;
+  using Args = pack<A...>;
+};
+
 } // namespace sfinae
 } // namespace ens
 
@@ -133,7 +154,9 @@ struct NAME                                                                    \
 <                                                                              \
   T,                                                                           \
   sig,                                                                         \
-  std::integral_constant<bool, SigCheck<sig, &T::FUNC>::value>                 \
+  std::is_same<decltype(std::declval<T>().FUNC(std::declval<                   \
+      ens::sfinae::FunctionTypes<sig>::A...>()...)),                           \
+      ens::sfinae::FunctionTypes<sig>::Ret>::type>                             \
 > : std::true_type {};
 
 /**
@@ -257,9 +280,9 @@ struct NAME                                                                  \
     ENS_HAS_METHOD_FORM_BASE(ENS_SINGLE_ARG(METHOD), ENS_SINGLE_ARG(NAME), 7)
 
 /**
- * ENS_HAS_EXACT_METHOD_FORM generates a template that allows to check at compile
- * time whether a given class has a method of the requested form. For example,
- * for the following class
+ * ENS_HAS_EXACT_METHOD_FORM generates a template that allows to check at
+ * compile time whether a given class has a method of the requested form. For
+ * example, for the following class
  *
  * class A
  * {
@@ -291,37 +314,4 @@ struct NAME                                                                  \
 #define ENS_HAS_EXACT_METHOD_FORM(METHOD, NAME) \
     ENS_HAS_METHOD_FORM_BASE(ENS_SINGLE_ARG(METHOD), ENS_SINGLE_ARG(NAME), 0)
 
-/**
- * A version of ENS_HAS_METHOD_FORM() where the maximum number of extra arguments is
- * set to the default of 7.
- *
- * ENS_HAS_METHOD_FORM generates a template that allows to check at compile time
- * whether a given class has a method of the requested form. For example, for
- * the following class
- *
- * class A
- * {
- *  public:
- *   ...
- *   Train(const arma::mat&, const arma::Row<size_t>&, double);
- *   ...
- * };
- *
- * and the following form of Train methods
- *
- * template<typename Class, typename...Ts>
- * using TrainForm =
- *     void(Class::*)(const arma::mat&, const arma::Row<size_t>&, Ts...);
- *
- * we can check whether the class A has a Train method of the specified form:
- *
- * ENS_HAS_METHOD_FORM(Train, HasTrain);
- * static_assert(HasTrain<A, TrainFrom>::value, "value should be true");
- *
- * The implementation is analogous to implementation of the macro ENS_HAS_MEM_FUNC.
- *
- * @param METHOD The name of the method to check for.
- * @param NAME The name of the struct to construct.
- */
-
 #endif

include/ensmallen_bits/function/static_checks.hpp

@@ -25,13 +25,16 @@ namespace traits {
  *
  * This is required by the FunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckEvaluate
 {
   const static bool value =
-      HasEvaluate<FunctionType, EvaluateForm>::value ||
-      HasEvaluate<FunctionType, EvaluateConstForm>::value ||
-      HasEvaluate<FunctionType, EvaluateStaticForm>::value;
+      HasEvaluate<FunctionType,
+          TypedForms<MatType, GradType>::template EvaluateForm>::value ||
+      HasEvaluate<FunctionType,
+          TypedForms<MatType, GradType>::template EvaluateConstForm>::value ||
+      HasEvaluate<FunctionType,
+          TypedForms<MatType, GradType>::template EvaluateStaticForm>::value;
 };
 
 /**
@@ -39,13 +42,16 @@ struct CheckEvaluate
  *
  * This is required by the FunctionType API.
  */
-template <typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckGradient
 {
   const static bool value =
-      HasGradient<FunctionType, GradientForm>::value ||
-      HasGradient<FunctionType, GradientConstForm>::value ||
-      HasGradient<FunctionType, GradientStaticForm>::value;
+      HasGradient<FunctionType,
+          TypedForms<MatType, GradType>::template GradientForm>::value ||
+      HasGradient<FunctionType,
+          TypedForms<MatType, GradType>::template GradientConstForm>::value ||
+      HasGradient<FunctionType,
+          TypedForms<MatType, GradType>::template GradientStaticForm>::value;
 };
 
 /**
@@ -53,13 +59,16 @@ struct CheckGradient
  *
  * This is required by the DecomposableFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckNumFunctions
 {
   const static bool value =
-      HasNumFunctions<FunctionType, NumFunctionsForm>::value ||
-      HasNumFunctions<FunctionType, NumFunctionsConstForm>::value ||
-      HasNumFunctions<FunctionType, NumFunctionsStaticForm>::value;
+      HasNumFunctions<FunctionType, TypedForms<MatType, GradType>::template
+          NumFunctionsForm>::value ||
+      HasNumFunctions<FunctionType, TypedForms<MatType, GradType>::template
+          NumFunctionsConstForm>::value ||
+      HasNumFunctions<FunctionType, TypedForms<MatType, GradType>::template
+          NumFunctionsStaticForm>::value;
 };
 
 /**
@@ -67,13 +76,16 @@ struct CheckNumFunctions
  *
  * This is required by the DecomposableFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckShuffle
 {
   const static bool value =
-      HasShuffle<FunctionType, ShuffleForm>::value ||
-      HasShuffle<FunctionType, ShuffleConstForm>::value ||
-      HasShuffle<FunctionType, ShuffleStaticForm>::value;
+      HasShuffle<FunctionType, TypedForms<MatType, GradType>::template
+          ShuffleForm>::value ||
+      HasShuffle<FunctionType, TypedForms<MatType, GradType>::template
+          ShuffleConstForm>::value ||
+      HasShuffle<FunctionType, TypedForms<MatType, GradType>::template
+          ShuffleStaticForm>::value;
 };
 
 /**
@@ -81,13 +93,16 @@ struct CheckShuffle
  *
  * This is required by the DecomposableFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckDecomposableEvaluate
 {
   const static bool value =
-      HasEvaluate<FunctionType, DecomposableEvaluateForm>::value ||
-      HasEvaluate<FunctionType, DecomposableEvaluateConstForm>::value ||
-      HasEvaluate<FunctionType, DecomposableEvaluateStaticForm>::value;
+      HasEvaluate<FunctionType, TypedForms<MatType, GradType>::template
+          DecomposableEvaluateForm>::value ||
+      HasEvaluate<FunctionType, TypedForms<MatType, GradType>::template
+          DecomposableEvaluateConstForm>::value ||
+      HasEvaluate<FunctionType, TypedForms<MatType, GradType>::template
+          DecomposableEvaluateStaticForm>::value;
 };
 
 /**
@@ -95,13 +110,16 @@ struct CheckDecomposableEvaluate
  *
  * This is required by the DecomposableFunctionType API.
  */
-template <typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckDecomposableGradient
 {
   const static bool value =
-      HasGradient<FunctionType, DecomposableGradientForm>::value ||
-      HasGradient<FunctionType, DecomposableGradientConstForm>::value ||
-      HasGradient<FunctionType, DecomposableGradientStaticForm>::value;
+      HasGradient<FunctionType, TypedForms<MatType, GradType>::template
+          DecomposableGradientForm>::value ||
+      HasGradient<FunctionType, TypedForms<MatType, GradType>::template
+          DecomposableGradientConstForm>::value ||
+      HasGradient<FunctionType, TypedForms<MatType, GradType>::template
+          DecomposableGradientStaticForm>::value;
 };
 
 /**
@@ -109,13 +127,16 @@ struct CheckDecomposableGradient
  *
  * This is required by the ConstrainedFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckNumConstraints
 {
   const static bool value =
-      HasNumConstraints<FunctionType, NumConstraintsForm>::value ||
-      HasNumConstraints<FunctionType, NumConstraintsConstForm>::value ||
-      HasNumConstraints<FunctionType, NumConstraintsStaticForm>::value;
+      HasNumConstraints<FunctionType, TypedForms<MatType, GradType>::template
+          NumConstraintsForm>::value ||
+      HasNumConstraints<FunctionType, TypedForms<MatType, GradType>::template
+          NumConstraintsConstForm>::value ||
+      HasNumConstraints<FunctionType, TypedForms<MatType, GradType>::template
+          NumConstraintsStaticForm>::value;
 };
 
 /**
@@ -123,13 +144,19 @@ struct CheckNumConstraints
  *
  * This is required by the ConstrainedFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckEvaluateConstraint
 {
   const static bool value =
-      HasEvaluateConstraint<FunctionType, EvaluateConstraintForm>::value ||
-      HasEvaluateConstraint<FunctionType, EvaluateConstraintConstForm>::value ||
-      HasEvaluateConstraint<FunctionType, EvaluateConstraintStaticForm>::value;
+      HasEvaluateConstraint<FunctionType,
+          TypedForms<MatType, GradType>::template
+              EvaluateConstraintForm>::value ||
+      HasEvaluateConstraint<FunctionType,
+          TypedForms<MatType, GradType>::template
+              EvaluateConstraintConstForm>::value ||
+      HasEvaluateConstraint<FunctionType,
+          TypedForms<MatType, GradType>::template
+              EvaluateConstraintStaticForm>::value;
 };
 
 /**
@@ -137,13 +164,19 @@ struct CheckEvaluateConstraint
  *
  * This is required by the ConstrainedFunctionType API.
  */
-template <typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckGradientConstraint
 {
   const static bool value =
-      HasGradientConstraint<FunctionType, GradientConstraintForm>::value ||
-      HasGradientConstraint<FunctionType, GradientConstraintConstForm>::value ||
-      HasGradientConstraint<FunctionType, GradientConstraintStaticForm>::value;
+      HasGradientConstraint<FunctionType,
+          TypedForms<MatType, GradType>::template
+              GradientConstraintForm>::value ||
+      HasGradientConstraint<FunctionType,
+          TypedForms<MatType, GradType>::template
+              GradientConstraintConstForm>::value ||
+      HasGradientConstraint<FunctionType,
+          TypedForms<MatType, GradType>::template
+              GradientConstraintStaticForm>::value;
 };
 
 /**
@@ -152,13 +185,16 @@ struct CheckGradientConstraint
  *
  * This is required by the SparseFunctionType API.
  */
-template <typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckSparseGradient
 {
   const static bool value =
-      HasGradient<FunctionType, SparseGradientForm>::value ||
-      HasGradient<FunctionType, SparseGradientConstForm>::value ||
-      HasGradient<FunctionType, SparseGradientStaticForm>::value;
+      HasGradient<FunctionType, TypedForms<MatType, GradType>::template
+          SparseGradientForm>::value ||
+      HasGradient<FunctionType, TypedForms<MatType, GradType>::template
+          SparseGradientConstForm>::value ||
+      HasGradient<FunctionType, TypedForms<MatType, GradType>::template
+          SparseGradientStaticForm>::value;
 };
 
 /**
@@ -166,13 +202,16 @@ struct CheckSparseGradient
  *
  * This is required by the ResolvableFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckNumFeatures
 {
   const static bool value =
-      HasNumFeatures<FunctionType, NumFeaturesForm>::value ||
-      HasNumFeatures<FunctionType, NumFeaturesConstForm>::value ||
-      HasNumFeatures<FunctionType, NumFeaturesStaticForm>::value;
+      HasNumFeatures<FunctionType, TypedForms<MatType, GradType>::template
+          NumFeaturesForm>::value ||
+      HasNumFeatures<FunctionType, TypedForms<MatType, GradType>::template
+          NumFeaturesConstForm>::value ||
+      HasNumFeatures<FunctionType, TypedForms<MatType, GradType>::template
+          NumFeaturesStaticForm>::value;
 };
 
 /**
@@ -180,13 +219,16 @@ struct CheckNumFeatures
  *
  * This is required by the ResolvableFunctionType API.
  */
-template <typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckPartialGradient
 {
   const static bool value =
-      HasPartialGradient<FunctionType, PartialGradientForm>::value ||
-      HasPartialGradient<FunctionType, PartialGradientConstForm>::value ||
-      HasPartialGradient<FunctionType, PartialGradientStaticForm>::value;
+      HasPartialGradient<FunctionType, TypedForms<MatType, GradType>::template
+          PartialGradientForm>::value ||
+      HasPartialGradient<FunctionType, TypedForms<MatType, GradType>::template
+          PartialGradientConstForm>::value ||
+      HasPartialGradient<FunctionType, TypedForms<MatType, GradType>::template
+          PartialGradientStaticForm>::value;
 };
 
 /**
@@ -194,15 +236,19 @@ struct CheckPartialGradient
  *
  * This is required by the FunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckEvaluateWithGradient
 {
   const static bool value =
-      HasEvaluateWithGradient<FunctionType, EvaluateWithGradientForm>::value ||
       HasEvaluateWithGradient<FunctionType,
-          EvaluateWithGradientConstForm>::value ||
+          TypedForms<MatType, GradType>::template
+              EvaluateWithGradientForm>::value ||
       HasEvaluateWithGradient<FunctionType,
-          EvaluateWithGradientStaticForm>::value;
+          TypedForms<MatType, GradType>::template
+              EvaluateWithGradientConstForm>::value ||
+      HasEvaluateWithGradient<FunctionType,
+          TypedForms<MatType, GradType>::template
+              EvaluateWithGradientStaticForm>::value;
 };
 
 /**
@@ -211,190 +257,218 @@ struct CheckEvaluateWithGradient
  *
  * This is required by the FunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckDecomposableEvaluateWithGradient
 {
   const static bool value =
       HasEvaluateWithGradient<FunctionType,
-          DecomposableEvaluateWithGradientForm>::value ||
+          TypedForms<MatType, GradType>::template
+              DecomposableEvaluateWithGradientForm>::value ||
       HasEvaluateWithGradient<FunctionType,
-          DecomposableEvaluateWithGradientConstForm>::value ||
+          TypedForms<MatType, GradType>::template
+              DecomposableEvaluateWithGradientConstForm>::value ||
       HasEvaluateWithGradient<FunctionType,
-          DecomposableEvaluateWithGradientStaticForm>::value;
+          TypedForms<MatType, GradType>::template
+              DecomposableEvaluateWithGradientStaticForm>::value;
 };
 
 /**
  * Perform checks for the regular FunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 inline void CheckFunctionTypeAPI()
 {
-  static_assert(CheckEvaluate<FunctionType>::value,
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(CheckEvaluate<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of Evaluate(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the FunctionType API; see the optimizer tutorial for details.");
 
-  static_assert(CheckGradient<FunctionType>::value,
+  static_assert(CheckGradient<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of Gradient(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the FunctionType API; see the optimizer tutorial for details.");
 
-  static_assert(CheckEvaluateWithGradient<FunctionType>::value,
+  static_assert(
+      CheckEvaluateWithGradient<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of "
       "EvaluateWithGradient().  Please check that the FunctionType fully "
       "satisfies the requirements of the FunctionType API; see the optimizer "
       "tutorial for more details.");
+#endif
 }
 
 /**
  * Perform checks for the DecomposableFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 inline void CheckDecomposableFunctionTypeAPI()
 {
-  static_assert(CheckDecomposableEvaluate<FunctionType>::value,
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(CheckDecomposableEvaluate<FunctionType,
+                                          MatType,
+                                          GradType>::value,
       "The FunctionType does not have a correct definition of a decomposable "
       "Evaluate() method.  Please check that the FunctionType fully satisfies"
       " the requirements of the DecomposableFunctionType API; see the optimizer"
       " tutorial for more details.");
 
-  static_assert(CheckDecomposableGradient<FunctionType>::value,
+  static_assert(CheckDecomposableGradient<FunctionType,
+                                          MatType,
+                                          GradType>::value,
       "The FunctionType does not have a correct definition of a decomposable "
       "Gradient() method.  Please check that the FunctionType fully satisfies"
       " the requirements of the DecomposableFunctionType API; see the optimizer"
       " tutorial for more details.");
 
-  static_assert(CheckDecomposableEvaluateWithGradient<FunctionType>::value,
+  static_assert(CheckDecomposableEvaluateWithGradient<FunctionType,
+                                                      MatType,
+                                                      GradType>::value,
       "The FunctionType does not have a correct definition of a decomposable "
       "EvaluateWithGradient() method.  Please check that the FunctionType "
       "fully satisfies the requirements of the DecomposableFunctionType API; "
       "see the optimizer tutorial for more details.");
 
-  static_assert(CheckNumFunctions<FunctionType>::value,
+  static_assert(CheckNumFunctions<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of NumFunctions(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the DecomposableFunctionType API; see the optimizer tutorial for more "
       "details.");
 
-  static_assert(CheckShuffle<FunctionType>::value,
+  static_assert(CheckShuffle<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of Shuffle(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the DecomposableFunctionType API; see the optimizer tutorial for more "
       "details.");
+#endif
 }
 
 /**
  * Perform checks for the SparseFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 inline void CheckSparseFunctionTypeAPI()
 {
-  static_assert(CheckNumFunctions<FunctionType>::value,
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(CheckNumFunctions<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of NumFunctions(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the SparseFunctionType API; see the optimizer tutorial for more "
       "details.");
 
-  static_assert(CheckDecomposableEvaluate<FunctionType>::value,
+  static_assert(CheckDecomposableEvaluate<FunctionType,
+                                          MatType,
+                                          GradType>::value,
       "The FunctionType does not have a correct definition of Evaluate(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the SparseFunctionType API; see the optimizer tutorial for more "
       "details.");
 
-  static_assert(CheckSparseGradient<FunctionType>::value,
+  static_assert(CheckSparseGradient<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of a sparse "
       "Gradient() method. Please check that the FunctionType fully satisfies "
       "the requirements of the SparseFunctionType API; see the optimizer "
       "tutorial for more details.");
+#endif
 }
 
 /**
  * Perform checks for the NonDifferentiableFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType>
 inline void CheckNonDifferentiableFunctionTypeAPI()
 {
-  static_assert(CheckEvaluate<FunctionType>::value,
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(CheckEvaluate<FunctionType, MatType, MatType>::value,
       "The FunctionType does not have a correct definition of Evaluate(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the NonDifferentiableFunctionType API; see the optimizer tutorial for "
       "more details.");
+#endif
 }
 
 /**
  * Perform checks for the ResolvableFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 inline void CheckResolvableFunctionTypeAPI()
 {
-  static_assert(CheckNumFeatures<FunctionType>::value,
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(CheckNumFeatures<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of NumFeatures(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the ResolvableFunctionType API; see the optimizer tutorial for more "
       "details.");
 
-  static_assert(CheckEvaluate<FunctionType>::value,
+  static_assert(CheckEvaluate<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of Evaluate(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the ResolvableFunctionType API; see the optimizer tutorial for more "
       "details.");
 
-  static_assert(CheckPartialGradient<FunctionType>::value,
+  static_assert(CheckPartialGradient<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of a partial "
       "Gradient() function. Please check that the FunctionType fully satisfies "
       "the requirements of the ResolvableFunctionType API; see the optimizer "
       "tutorial for more details.");
+#endif
 }
 
 /**
  * Perform checks for the ConstrainedFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 inline void CheckConstrainedFunctionTypeAPI()
 {
-  static_assert(CheckEvaluate<FunctionType>::value,
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(CheckEvaluate<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of Evaluate(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the ConstrainedFunctionType API; see the optimizer tutorial for more "
       "details.");
 
-  static_assert(CheckGradient<FunctionType>::value,
+  static_assert(CheckGradient<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of Gradient(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the ConstrainedFunctionType API; see the optimizer tutorial for more "
       "details.");
 
-  static_assert(CheckNumConstraints<FunctionType>::value,
+  static_assert(CheckNumConstraints<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of NumConstraints()."
       " Please check that the FunctionType fully satisfies the requirements of "
       "the ConstrainedFunctionType API; see the optimizer tutorial for more "
       "details.");
 
-  static_assert(CheckEvaluateConstraint<FunctionType>::value,
+  static_assert(CheckEvaluateConstraint<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of "
       "EvaluateConstraint(). Please check that the FunctionType fully satisfies"
       " the ConstrainedFunctionType API; see the optimizer tutorial for more "
       "details.");
 
-  static_assert(CheckGradientConstraint<FunctionType>::value,
+  static_assert(CheckGradientConstraint<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of "
       "GradientConstraint(). Please check that the FunctionType fully satisfies"
       " the ConstrainedFunctionType API; see the optimizer tutorial for more "
       "details.");
+#endif
 }
 
 /**
  * Perform checks for the NonDifferentiableDecomposableFunctionType API.  (I
  * know, it is a long name...)
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType>
 inline void CheckNonDifferentiableDecomposableFunctionTypeAPI()
 {
-  static_assert(CheckDecomposableEvaluate<FunctionType>::value,
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(CheckDecomposableEvaluate<FunctionType,
+                                          MatType,
+                                          MatType>::value,
       "The FunctionType does not have a correct definition of Evaluate(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the NonDifferentiableDecomposableFunctionType API; see the optimizer "
       "tutorial for more details.");
+#endif
 }
 
 } // namespace traits

include/ensmallen_bits/function/traits.hpp

@@ -14,6 +14,7 @@
 #define ENSMALLEN_FUNCTION_TRAITS_HPP
 
 #include "sfinae_utility.hpp"
+#include "arma_traits.hpp"
 
 namespace ens {
 namespace traits {
@@ -39,210 +40,246 @@ ENS_HAS_EXACT_METHOD_FORM(NumFeatures, HasNumFeatures)
 //! Detect a PartialGradient() method.
 ENS_HAS_EXACT_METHOD_FORM(PartialGradient, HasPartialGradient)
 
-//! This is the form of a non-const Evaluate() method.
-template<typename FunctionType>
-using EvaluateForm = double(FunctionType::*)(const arma::mat&);
+template<typename MatType, typename GradType>
+struct TypedForms
+{
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
 
-//! This is the form of a const Evaluate() method.
-template<typename FunctionType>
-using EvaluateConstForm =
-    double(FunctionType::*)(const arma::mat&) const;
+  //! This is the form of a non-const Evaluate() method.
+  template<typename FunctionType>
+  using EvaluateForm =
+      typename BaseMatType::elem_type(FunctionType::*)(const BaseMatType&);
 
-//! This is the form of a static Evaluate() method.
-template<typename FunctionType>
-using EvaluateStaticForm = double(*)(const arma::mat&);
+  //! This is the form of a const Evaluate() method.
+  template<typename FunctionType>
+  using EvaluateConstForm = typename BaseMatType::elem_type(FunctionType::*)(
+      const BaseMatType&) const;
 
-//! This is the form of a non-const Gradient() method.
-template<typename FunctionType>
-using GradientForm = void(FunctionType::*)(const arma::mat&, arma::mat&);
+  //! This is the form of a static Evaluate() method.
+  template<typename FunctionType>
+  using EvaluateStaticForm = typename BaseMatType::elem_type(*)(
+      const BaseMatType&);
 
-//! This is the form of a const Gradient() method.
-template<typename FunctionType>
-using GradientConstForm =
-    void(FunctionType::*)(const arma::mat&, arma::mat&) const;
+  //! This is the form of a non-const Gradient() method.
+  template<typename FunctionType>
+  using GradientForm = void(FunctionType::*)(const BaseMatType&, BaseGradType&);
 
-//! This is the form of a static Gradient() method.
-template<typename FunctionType>
-using GradientStaticForm = void(*)(const arma::mat&, arma::mat&);
+  //! This is the form of a const Gradient() method.
+  template<typename FunctionType>
+  using GradientConstForm =
+      void(FunctionType::*)(const BaseMatType&, BaseGradType&) const;
 
-//! This is the form of a non-const EvaluateWithGradient() method.
-template<typename FunctionType>
-using EvaluateWithGradientForm =
-    double(FunctionType::*)(const arma::mat&, arma::mat&);
+  //! This is the form of a static Gradient() method.
+  template<typename FunctionType>
+  using GradientStaticForm = void(*)(const BaseMatType&, BaseGradType&);
 
-//! This is the form of a const EvaluateWithGradient() method.
-template<typename FunctionType>
-using EvaluateWithGradientConstForm =
-    double(FunctionType::*)(const arma::mat&, arma::mat&) const;
+  //! This is the form of a non-const EvaluateWithGradient() method.
+  template<typename FunctionType>
+  using EvaluateWithGradientForm =
+      typename BaseMatType::elem_type(FunctionType::*)(const BaseMatType&,
+                                                       BaseGradType&);
 
-//! This is the form of a static EvaluateWithGradient() method.
-template<typename FunctionType>
-using EvaluateWithGradientStaticForm =
-    double(*)(const arma::mat&, arma::mat&);
+  //! This is the form of a const EvaluateWithGradient() method.
+  template<typename FunctionType>
+  using EvaluateWithGradientConstForm =
+      typename BaseMatType::elem_type(FunctionType::*)(const BaseMatType&,
+                                                       BaseGradType&) const;
 
-//! This is the form of a non-const NumFunctions() method.
-template <typename FunctionType>
-using NumFunctionsForm = size_t(FunctionType::*)();
+  //! This is the form of a static EvaluateWithGradient() method.
+  template<typename FunctionType>
+  using EvaluateWithGradientStaticForm = typename BaseMatType::elem_type(*)(
+      const BaseMatType&, BaseGradType&);
 
-//! This is the form of a const NumFunctions() method.
-template <typename FunctionType>
-using NumFunctionsConstForm = size_t(FunctionType::*)() const;
+  //! This is the form of a non-const NumFunctions() method.
+  template <typename FunctionType>
+  using NumFunctionsForm = size_t(FunctionType::*)();
 
-//! This is the form of a static NumFunctions() method.
-template<typename FunctionType>
-using NumFunctionsStaticForm = size_t(*)();
+  //! This is the form of a const NumFunctions() method.
+  template <typename FunctionType>
+  using NumFunctionsConstForm = size_t(FunctionType::*)() const;
 
-//! This is the form of a non-const Shuffle() method.
-template<typename FunctionType>
-using ShuffleForm = void(FunctionType::*)();
+  //! This is the form of a static NumFunctions() method.
+  template<typename FunctionType>
+  using NumFunctionsStaticForm = size_t(*)();
 
-//! This is the form of a const Shuffle() method.
-template<typename FunctionType>
-using ShuffleConstForm = void(FunctionType::*)() const;
+  //! This is the form of a non-const Shuffle() method.
+  template<typename FunctionType>
+  using ShuffleForm = void(FunctionType::*)();
 
-//! This is the form of a static Shuffle() method.
-template<typename FunctionType>
-using ShuffleStaticForm = void(*)();
+  //! This is the form of a const Shuffle() method.
+  template<typename FunctionType>
+  using ShuffleConstForm = void(FunctionType::*)() const;
 
-//! This is the form of a decomposable Evaluate() method.
-template<typename FunctionType>
-using DecomposableEvaluateForm = double(FunctionType::*)(
-    const arma::mat&, const size_t, const size_t);
+  //! This is the form of a static Shuffle() method.
+  template<typename FunctionType>
+  using ShuffleStaticForm = void(*)();
 
-//! This is the form of a decomposable const Evaluate() method.
-template<typename FunctionType>
-using DecomposableEvaluateConstForm = double(FunctionType::*)(
-    const arma::mat&, const size_t, const size_t) const;
+  //! This is the form of a decomposable Evaluate() method.
+  template<typename FunctionType>
+  using DecomposableEvaluateForm =
+      typename BaseMatType::elem_type(FunctionType::*)(const BaseMatType&,
+                                                       const size_t,
+                                                       const size_t);
 
-//! This is the form of a decomposable static Evaluate() method.
-template<typename FunctionType>
-using DecomposableEvaluateStaticForm = double(*)(
-    const arma::mat&, const size_t, const size_t);
+  //! This is the form of a decomposable const Evaluate() method.
+  template<typename FunctionType>
+  using DecomposableEvaluateConstForm =
+      typename BaseMatType::elem_type(FunctionType::*)(const BaseMatType&,
+                                                       const size_t,
+                                                       const size_t) const;
 
-//! This is the form of a decomposable non-const Gradient() method.
-template<typename FunctionType>
-using DecomposableGradientForm = void(FunctionType::*)(
-    const arma::mat&, const size_t, arma::mat&, const size_t);
+  //! This is the form of a decomposable static Evaluate() method.
+  template<typename FunctionType>
+  using DecomposableEvaluateStaticForm = typename BaseMatType::elem_type(*)(
+        const BaseMatType&, const size_t, const size_t);
 
-//! This the form of a decomposable const Gradient() method.
-template<typename FunctionType>
-using DecomposableGradientConstForm = void(FunctionType::*)(
-    const arma::mat&, const size_t, arma::mat&, const size_t) const;
+  //! This is the form of a decomposable non-const Gradient() method.
+  template<typename FunctionType>
+  using DecomposableGradientForm = void(FunctionType::*)(
+      const BaseMatType&, const size_t, BaseGradType&, const size_t);
 
-//! This is the form of a decomposable static Gradient() method.
-template<typename FunctionType>
-using DecomposableGradientStaticForm = void(*)(
-    const arma::mat&, const size_t, arma::mat&, const size_t);
+  //! This the form of a decomposable const Gradient() method.
+  template<typename FunctionType>
+  using DecomposableGradientConstForm = void(FunctionType::*)(
+      const BaseMatType&, const size_t, BaseGradType&, const size_t) const;
 
-//! This is the form of a decomposable non-const EvaluateWithGradient() method.
-template<typename FunctionType>
-using DecomposableEvaluateWithGradientForm = double(FunctionType::*)(
-    const arma::mat&, const size_t, arma::mat&, const size_t);
+  //! This is the form of a decomposable static Gradient() method.
+  template<typename FunctionType>
+  using DecomposableGradientStaticForm = void(*)(
+      const BaseMatType&, const size_t, BaseGradType&, const size_t);
 
-//! This is the form of a decomposable const EvaluateWithGradient() method.
-template<typename FunctionType>
-using DecomposableEvaluateWithGradientConstForm = double(FunctionType::*)(
-    const arma::mat&, const size_t, arma::mat&, const size_t) const;
+  //! This is the form of a decomposable non-const EvaluateWithGradient()
+  //! method.
+  template<typename FunctionType>
+  using DecomposableEvaluateWithGradientForm =
+      typename BaseMatType::elem_type(FunctionType::*)(const BaseMatType&,
+                                                       const size_t,
+                                                       BaseGradType&,
+                                                       const size_t);
 
-//! This is the form of a decomposable static EvaluateWithGradient() method.
-template<typename FunctionType>
-using DecomposableEvaluateWithGradientStaticForm = double(*)(
-    const arma::mat&, const size_t, arma::mat&, const size_t);
+  //! This is the form of a decomposable const EvaluateWithGradient() method.
+  template<typename FunctionType>
+  using DecomposableEvaluateWithGradientConstForm =
+      typename BaseMatType::elem_type(FunctionType::*)(const BaseMatType&,
+                                                       const size_t,
+                                                       BaseGradType&,
+                                                       const size_t) const;
 
-//! This is the form of a non-const NumConstraints() method.
-template<typename FunctionType>
-using NumConstraintsForm = size_t(FunctionType::*)();
+  //! This is the form of a decomposable static EvaluateWithGradient() method.
+  template<typename FunctionType>
+  using DecomposableEvaluateWithGradientStaticForm =
+      typename BaseMatType::elem_type(*)(const BaseMatType&,
+                                         const size_t,
+                                         BaseGradType&,
+                                         const size_t);
 
-//! This is the form of a const NumConstraints() method.
-template<typename FunctionType>
-using NumConstraintsConstForm = size_t(FunctionType::*)() const;
+  //! This is the form of a non-const NumConstraints() method.
+  template<typename FunctionType>
+  using NumConstraintsForm = size_t(FunctionType::*)();
 
-//! This is the form of a static NumConstraints() method.
-template<typename FunctionType>
-using NumConstraintsStaticForm = size_t(*)();
+  //! This is the form of a const NumConstraints() method.
+  template<typename FunctionType>
+  using NumConstraintsConstForm = size_t(FunctionType::*)() const;
 
-//! This is the form of a non-const EvaluateConstraint() method.
-template <typename FunctionType>
-using EvaluateConstraintForm = double(FunctionType::*)(
-    const size_t, const arma::mat&);
+  //! This is the form of a static NumConstraints() method.
+  template<typename FunctionType>
+  using NumConstraintsStaticForm = size_t(*)();
 
-//! This is the form of a const EvaluateConstraint() method.
-template<typename FunctionType>
-using EvaluateConstraintConstForm = double(FunctionType::*)(
-    const size_t, const arma::mat&) const;
+  //! This is the form of a non-const EvaluateConstraint() method.
+  template <typename FunctionType>
+  using EvaluateConstraintForm =
+      typename BaseMatType::elem_type(FunctionType::*)(const size_t,
+                                                       const BaseMatType&);
 
-//! This is the form of a static EvaluateConstraint() method.
-template<typename FunctionType>
-using EvaluateConstraintStaticForm = double(*)(const size_t, const arma::mat&);
+  //! This is the form of a const EvaluateConstraint() method.
+  template<typename FunctionType>
+  using EvaluateConstraintConstForm =
+      typename BaseMatType::elem_type(FunctionType::*)(const size_t,
+                                                       const BaseMatType&)
+          const;
 
-//! This is the form of a non-const GradientConstraint() method.
-template <typename FunctionType>
-using GradientConstraintForm = void(FunctionType::*)(
-    const size_t, const arma::mat&, arma::mat&);
+  //! This is the form of a static EvaluateConstraint() method.
+  template<typename FunctionType>
+  using EvaluateConstraintStaticForm = typename BaseMatType::elem_type(*)(
+      const size_t, const BaseMatType&);
 
-//! This is the form of a const GradientConstraint() method.
-template<typename FunctionType>
-using GradientConstraintConstForm = void(FunctionType::*)(
-    const size_t, const arma::mat&, arma::mat&) const;
+  //! This is the form of a non-const GradientConstraint() method.
+  template <typename FunctionType>
+  using GradientConstraintForm = void(FunctionType::*)(
+      const size_t, const BaseMatType&, BaseGradType&);
 
-//! This is the form of a static GradientConstraint() method.
-template<typename Class, typename... Ts>
-using GradientConstraintStaticForm = void(*)(
-    const size_t, const arma::mat&, arma::mat&);
+  //! This is the form of a const GradientConstraint() method.
+  template<typename FunctionType>
+  using GradientConstraintConstForm = void(FunctionType::*)(
+      const size_t, const BaseMatType&, BaseGradType&) const;
 
-//! This is the form of a non-const sparse Gradient() method.
-template<typename FunctionType>
-using SparseGradientForm = void(FunctionType::*)(
-    const arma::mat&, const size_t, arma::sp_mat&, const size_t);
+  //! This is the form of a static GradientConstraint() method.
+  template<typename Class, typename... Ts>
+  using GradientConstraintStaticForm = void(*)(
+      const size_t, const BaseMatType&, BaseGradType&);
 
-//! This is the form of a const sparse Gradient() method.
-template<typename FunctionType>
-using SparseGradientConstForm = void(FunctionType::*)(
-    const arma::mat&, const size_t, arma::sp_mat&, const size_t) const;
+  //! This is the form of a non-const sparse Gradient() method.
+  //! This check isn't particularly useful---the user needs to specify a sparse
+  //! gradient type...
+  template<typename FunctionType>
+  using SparseGradientForm = void(FunctionType::*)(
+      const BaseMatType&, const size_t, BaseGradType&, const size_t);
 
-//! This is the form of a static sparse Gradient() method.
-template<typename FunctionType>
-using SparseGradientStaticForm = void(*)(
-    const arma::mat&, const size_t, arma::sp_mat&, const size_t);
+  //! This is the form of a const sparse Gradient() method.
+  //! This check isn't particularly useful---the user needs to specify a sparse
+  //! gradient type...
+  template<typename FunctionType>
+  using SparseGradientConstForm = void(FunctionType::*)(
+      const BaseMatType&, const size_t, BaseGradType&, const size_t) const;
 
-//! This is the form of a non-const NumFeatures() method.
-template<typename FunctionType>
-using NumFeaturesForm = size_t(FunctionType::*)();
+  //! This is the form of a static sparse Gradient() method.
+  //! This check isn't particularly useful---the user needs to specify a sparse
+  //! gradient type...
+  template<typename FunctionType>
+  using SparseGradientStaticForm = void(*)(
+      const BaseMatType&, const size_t, BaseGradType&, const size_t);
 
-//! This is the form of a const NumFeatures() method.
-template<typename FunctionType>
-using NumFeaturesConstForm = size_t(FunctionType::*)() const;
+  //! This is the form of a non-const NumFeatures() method.
+  template<typename FunctionType>
+  using NumFeaturesForm = size_t(FunctionType::*)();
 
-//! This is the form of a static NumFeatures() method.
-template<typename FunctionType>
-using NumFeaturesStaticForm = size_t(*)();
+  //! This is the form of a const NumFeatures() method.
+  template<typename FunctionType>
+  using NumFeaturesConstForm = size_t(FunctionType::*)() const;
 
-//! This is the form of a non-const PartialGradient() method.
-template<typename FunctionType>
-using PartialGradientForm = void(FunctionType::*)(
-    const arma::mat&, const size_t, arma::sp_mat&);
+  //! This is the form of a static NumFeatures() method.
+  template<typename FunctionType>
+  using NumFeaturesStaticForm = size_t(*)();
 
-//! This is the form of a const PartialGradient() method.
-template<typename FunctionType>
-using PartialGradientConstForm = void(FunctionType::*)(
-    const arma::mat&, const size_t, arma::sp_mat&) const;
+  //! This is the form of a non-const PartialGradient() method.
+  template<typename FunctionType>
+  using PartialGradientForm = void(FunctionType::*)(
+      const BaseMatType&, const size_t, BaseGradType&);
 
-//! This is the form of a static PartialGradient() method.
-template<typename FunctionType>
-using PartialGradientStaticForm = void(*)(
-    const arma::mat&, const size_t, arma::sp_mat&);
+  //! This is the form of a const PartialGradient() method.
+  template<typename FunctionType>
+  using PartialGradientConstForm = void(FunctionType::*)(
+      const BaseMatType&, const size_t, BaseGradType&) const;
 
-//! This is a utility struct that will match any non-const form.
-template<typename FunctionType, typename... Ts>
-using OtherForm = double(FunctionType::*)(Ts...);
+  //! This is the form of a static PartialGradient() method.
+  template<typename FunctionType>
+  using PartialGradientStaticForm = void(*)(
+      const BaseMatType&, const size_t, BaseGradType&);
 
-//! This is a utility struct that will match any const form.
-template<typename FunctionType, typename... Ts>
-using OtherConstForm = double(FunctionType::*)(Ts...) const;
+  //! This is a utility struct that will match any non-const form.
+  template<typename FunctionType, typename... Ts>
+  using OtherForm = typename BaseMatType::elem_type(FunctionType::*)(Ts...);
 
-//! This is a utility struct that will match any static form.
-template<typename FunctionType, typename... Ts>
-using OtherStaticForm = double(*)(Ts...);
+  //! This is a utility struct that will match any const form.
+  template<typename FunctionType, typename... Ts>
+  using OtherConstForm = typename BaseMatType::elem_type(FunctionType::*)(Ts...)
+      const;
+
+  //! This is a utility struct that will match any static form.
+  template<typename FunctionType, typename... Ts>
+  using OtherStaticForm = typename BaseMatType::elem_type(*)(Ts...);
+};
 
 /**
  * This is a utility type used to provide unusable overloads from each of the
@@ -299,12 +336,12 @@ struct HasNonConstSignatures
 /**
  * Utility struct: sometimes we want to know if we have two functions available,
  * and that at least one of them is const and both of them are not non-const and
- * non-static.  If the corresponding checkers (from ENS_HAS_METHOD_FORM()) are given
- * as CheckerA and CheckerB, and the corresponding const and static function
- * signatures are given as ConstSignatureA, StaticSignatureA, ConstSignatureB,
- * and StaticSignatureB, then 'value' will be true if methods with the correct
- * names exist in the given ClassType and at least one of those two methods is
- * const, and neither method is non-const and non-static.
+ * non-static.  If the corresponding checkers (from ENS_HAS_METHOD_FORM()) are
+ * given as CheckerA and CheckerB, and the corresponding const and static
+ * function signatures are given as ConstSignatureA, StaticSignatureA,
+ * ConstSignatureB, and StaticSignatureB, then 'value' will be true if methods
+ * with the correct names exist in the given ClassType and at least one of those
+ * two methods is const, and neither method is non-const and non-static.
  */
 template<typename ClassType,
          template<typename, template<typename...> class, size_t> class CheckerA,

include/ensmallen_bits/fw/atoms.hpp

@@ -18,7 +18,8 @@ namespace ens {
 
 /**
  * Class to hold the information and operations of current atoms in the
- * soluton space.
+ * soluton space.  This is not fully templatized, and may cost some extra
+ * operations for the conversion.
  */
 class Atoms
 {
@@ -31,7 +32,7 @@ class Atoms
    * @param v new atom to be added.
    * @param c coefficient of the new atom.
    */
-  void AddAtom(const arma::vec& v, FuncSq& function, const double c = 0)
+  void AddAtom(const arma::mat& v, FuncSq& function, const double c = 0)
   {
     if (currentAtoms.is_empty())
     {
@@ -89,7 +90,7 @@ class Atoms
       // Solve for current gradient.
       arma::mat x;
       RecoverVector(x);
-      arma::mat gradient(size(x));
+      arma::mat gradient(arma::size(x));
       function.Gradient(x, gradient);
 
       // Find possible atom to be deleted.
@@ -113,7 +114,7 @@ class Atoms
       // add an atom norm constraint, you could use projected gradient method,
       // see the implementaton of ProjectedGradientEnhancement().
       arma::vec newCoeffs =
-          solve(function.MatrixA() * newAtoms, function.Vectorb());
+          solve(function.MatrixA() * newAtoms, function.Vectorb(), arma::solve_opts::fast);
 
       // Evaluate the function again.
       double Fnew = function.Evaluate(newAtoms * newCoeffs);

include/ensmallen_bits/fw/constr_lpball.hpp

@@ -79,36 +79,42 @@ class ConstrLpBallSolver
    * @param v Input local gradient.
    * @param s Output optimal solution in the constrained domain (lp ball).
    */
-  void Optimize(const arma::mat& v,
-                arma::mat& s)
+  template<typename MatType>
+  void Optimize(const MatType& v,
+                MatType& s)
   {
+    typedef typename MatType::elem_type ElemType;
+
     if (p == std::numeric_limits<double>::infinity())
     {
       // l-inf ball.
-      s = -sign(v);
+      s = -arma::sign(v);
       if (regFlag)
-        s = s / lambda;   // element-wise division.
+      {
+        // Do element-wise division.
+        s /= arma::conv_to<arma::Col<ElemType>>::from(lambda);
+      }
     }
     else if (p > 1.0)
     {
       // lp ball with 1<p<inf.
       if (regFlag)
-        s = v / lambda;
+        s = v / arma::conv_to<arma::Col<ElemType>>::from(lambda);
       else
         s = v;
 
       double q = 1 / (1.0 - 1.0 / p);
-      s = - sign(v) % pow(abs(s), q - 1);  // element-wise multiplication.
+      s = -arma::sign(v) % arma::pow(arma::abs(s), q - 1);
       s = arma::normalise(s, p);
 
       if (regFlag)
-        s = s / lambda;
+        s = s / arma::conv_to<arma::Col<ElemType>>::from(lambda);
     }
     else if (p == 1.0)
     {
       // l1 ball, also used in OMP.
       if (regFlag)
-        s = arma::abs(v / lambda);
+        s = arma::abs(v / arma::conv_to<arma::Col<ElemType>>::from(lambda));
       else
         s = arma::abs(v);
 
@@ -119,7 +125,7 @@ class ConstrLpBallSolver
       s(k) = -((0.0 < v(k)) - (v(k) < 0.0));
 
       if (regFlag)
-        s = s / lambda;
+        s = s / arma::conv_to<arma::Col<ElemType>>::from(lambda);
     }
     else
     {
@@ -135,14 +141,14 @@ class ConstrLpBallSolver
   double& P() { return p;}
 
   //! Get regularization flag.
-  bool RegFlag() const {return regFlag;}
+  bool RegFlag() const { return regFlag; }
   //! Modify regularization flag.
-  bool& RegFlag() {return regFlag;}
+  bool& RegFlag() { return regFlag; }
 
   //! Get the regularization parameter.
-  arma::vec Lambda() const {return lambda;}
+  arma::vec Lambda() const { return lambda; }
   //! Modify the regularization parameter.
-  arma::vec& Lambda() {return lambda;}
+  arma::vec& Lambda() { return lambda; }
 
  private:
   //! lp norm, 1<=p<=inf;

include/ensmallen_bits/fw/constr_structure_group.hpp

@@ -73,7 +73,7 @@ class ConstrStructGroupSolver
    *                       group, and compute norm in each group.
    */
   ConstrStructGroupSolver(GroupType& groupExtractor) :
-    groupExtractor(groupExtractor)
+      groupExtractor(groupExtractor)
   { /* Nothing to do */ }
 
   /**
@@ -82,18 +82,21 @@ class ConstrStructGroupSolver
    * @param v Input local gradient.
    * @param s Output optimal solution in the constrained atom domain.
    */
-  void Optimize(const arma::mat& v, arma::mat& s)
+  template<typename MatType>
+  void Optimize(const MatType& v, MatType& s)
   {
+    typedef typename MatType::elem_type ElemType;
+
     size_t nGroups = groupExtractor.NumGroups();
-    double dualNorm = 0;
+    ElemType dualNorm = 0;
     size_t optimalGroup = 1;
 
     // Find the optimal group.
     for (size_t i = 1; i <= nGroups; ++i)
     {
-      arma::vec y;
+      MatType y;
       groupExtractor.ProjectToGroup(v, i, y);
-      double newNorm = groupExtractor.DualNorm(y, i);
+      ElemType newNorm = groupExtractor.DualNorm(y, i);
 
       // Find the group with largest dual norm.
       if (newNorm > dualNorm)
@@ -142,11 +145,12 @@ class GroupLpBall
    * @param groupId input ID number of the group, start from 1.
    * @param y output projection of the vector to specific group.
    */
-  void ProjectToGroup(const arma::mat& v, const size_t groupId, arma::vec& y)
+  template<typename MatType>
+  void ProjectToGroup(const MatType& v, const size_t groupId, MatType& y)
   {
     arma::uvec& indList = groupIndicesList[groupId - 1];
     size_t dim = indList.n_elem;
-    y.set_size(dim);
+    y.set_size(dim, 1);
 
     for (size_t i = 0; i < dim; ++i)
       y(i) = v(indList(i));
@@ -160,14 +164,15 @@ class GroupLpBall
    * @param groupId optimal atom belongs to this group.
    * @param s output optimal atom.
    */
-  void OptimalFromGroup(const arma::mat& v, const size_t groupId, arma::mat& s)
+  template<typename MatType>
+  void OptimalFromGroup(const MatType& v, const size_t groupId, MatType& s)
   {
     // Project v to group.
-    arma::vec yk;
+    MatType yk;
     ProjectToGroup(v, groupId, yk);
 
     // Optimize in this group.
-    arma::vec sProj(yk.n_elem);
+    MatType sProj(yk.n_elem, 1);
     lpBallSolver.Optimize(yk, sProj);
 
     // Recover s to the original dimension.
@@ -190,7 +195,8 @@ class GroupLpBall
    * @param yk compute the q-norm of yk.
    * @param groupId group ID number.
    */
-  double DualNorm(const arma::vec& yk, const int groupId)
+  template<typename MatType>
+  typename MatType::elem_type DualNorm(const MatType& yk, const int groupId)
   {
     if (p == std::numeric_limits<double>::infinity())
     {
@@ -200,8 +206,8 @@ class GroupLpBall
     else if (p > 1.0)
     {
       // p norm, return q-norm
-      double q = 1.0 / (1.0 - 1.0/p);
-      return  arma::norm(yk, q);
+      double q = 1.0 / (1.0 - 1.0 / p);
+      return arma::norm(yk, q);
     }
     else if (p == 1.0)
     {
@@ -233,9 +239,6 @@ class GroupLpBall
   ConstrLpBallSolver lpBallSolver;
 };
 
-
 } // namespace ens
 
-
-
 #endif

include/ensmallen_bits/fw/frank_wolfe.hpp

@@ -114,13 +114,36 @@ class FrankWolfe
    *   void Gradient(const arma::mat& coordinates,
    *                 arma::mat& gradient);
    *
+   * @tparam FunctionType Type of function to be optimized.
+   * @tparam MatType Type of objective matrix.
+   * @tparam GradType Type of gradient matrix (default is MatType).
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to be optimized.
    * @param iterate Input with starting point, and will be modified to save
    *                the output optimial solution coordinates.
+   * @param callbacks Callback functions.
    * @return Objective value at the final solution.
    */
-  template<typename FunctionType>
-  double Optimize(FunctionType& function, arma::mat& iterate);
+  template<typename FunctionType, typename MatType, typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(FunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(FunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<FunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
 
   //! Get the linear constrained solver.
   const LinearConstrSolverType& LinearConstrSolver()

include/ensmallen_bits/fw/frank_wolfe_impl.hpp

@@ -39,34 +39,56 @@ FrankWolfe(const LinearConstrSolverType linearConstrSolver,
 template<
     typename LinearConstrSolverType,
     typename UpdateRuleType>
-template<typename FunctionType>
-double FrankWolfe<LinearConstrSolverType, UpdateRuleType>::
-Optimize(FunctionType& function, arma::mat& iterate)
+template<typename FunctionType, typename MatType, typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value,
+typename MatType::elem_type>::type
+FrankWolfe<LinearConstrSolverType, UpdateRuleType>::Optimize(
+  FunctionType& function,
+  MatType& iterateIn,
+  CallbackTypes&&... callbacks)
 {
-  typedef Function<FunctionType> FullFunctionType;
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
+  typedef Function<FunctionType, BaseMatType, BaseGradType> FullFunctionType;
   FullFunctionType& f = static_cast<FullFunctionType&>(function);
 
   // Make sure we have all necessary functions.
-  traits::CheckFunctionTypeAPI<FullFunctionType>();
+  traits::CheckFunctionTypeAPI<FullFunctionType, BaseMatType, BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
 
   // To keep track of the function value.
-  double currentObjective = DBL_MAX;
+  ElemType currentObjective = std::numeric_limits<ElemType>::max();
 
-  arma::mat gradient(iterate.n_rows, iterate.n_cols);
-  arma::mat s(iterate.n_rows, iterate.n_cols);
-  arma::mat iterateNew(iterate.n_rows, iterate.n_cols);
+  BaseGradType gradient(iterate.n_rows, iterate.n_cols);
+  BaseMatType s(iterate.n_rows, iterate.n_cols);
+  BaseMatType iterateNew(iterate.n_rows, iterate.n_cols);
   double gap = 0;
 
-  for (size_t i = 1; i != maxIterations; ++i)
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
+  terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  for (size_t i = 1; i != maxIterations && !terminate; ++i)
   {
     currentObjective = f.EvaluateWithGradient(iterate, gradient);
 
+    terminate |= Callback::EvaluateWithGradient(*this, f, iterate,
+        currentObjective, gradient, callbacks...);
+
     // Output current objective function.
     Info << "FrankWolfe::Optimize(): iteration " << i << ", objective "
         << currentObjective << "." << std::endl;
 
     // Solve linear constrained problem, solution saved in s.
-    linearConstrSolver.Optimize(gradient, s);
+    linearConstrSolver.Optimize(gradient, s, callbacks...);
 
     // Check duality gap for return condition.
     gap = std::fabs(dot(iterate - s, gradient));
@@ -74,18 +96,23 @@ Optimize(FunctionType& function, arma::mat& iterate)
     {
       Info << "FrankWolfe::Optimize(): minimized within tolerance "
           << tolerance << "; " << "terminating optimization." << std::endl;
+
+      Callback::EndOptimization(*this, f, iterate, callbacks...);
       return currentObjective;
     }
 
-
     // Update solution, save in iterateNew.
-    updateRule.Update(f, iterate, s, iterateNew, i);
+    updateRule.template Update<FunctionType, BaseMatType, BaseGradType>(f,
+        iterate, s, iterateNew, i);
 
     iterate = std::move(iterateNew);
+    terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
   }
 
   Info << "FrankWolfe::Optimize(): maximum iterations (" << maxIterations
       << ") reached; " << "terminating optimization." << std::endl;
+
+  Callback::EndOptimization(*this, f, iterate, callbacks...);
   return currentObjective;
 } // Optimize()
 

include/ensmallen_bits/fw/line_search/line_search.hpp

@@ -44,8 +44,12 @@ class LineSearch
    *           coordinate of the optimal solution.
    * @return Minimum solution function value.
    */
-  template<typename FunctionType>
-  double Optimize(FunctionType& function, const arma::mat& x1, arma::mat& x2);
+  template<typename FunctionType,
+           typename MatType,
+           typename GradType = MatType>
+  typename MatType::elem_type Optimize(FunctionType& function,
+                                       const MatType& x1,
+                                       MatType& x2);
 
   //! Get the maximum number of iterations (0 indicates no limit).
   size_t MaxIterations() const { return maxIterations; }
@@ -74,11 +78,11 @@ class LineSearch
    *
    * @return Derivative of function(x0 + gamma * deltaX) with respect to gamma.
    */
-  template<typename FunctionType>
-  double Derivative(FunctionType& function,
-                    const arma::mat& x0,
-                    const arma::mat& deltaX,
-                    const double gamma);
+  template<typename FunctionType, typename MatType, typename GradType>
+  typename MatType::elem_type Derivative(FunctionType& function,
+                                         const MatType& x0,
+                                         const MatType& deltaX,
+                                         const double gamma);
 };  // class LineSearch
 } // namespace ens
 

include/ensmallen_bits/fw/line_search/line_search_impl.hpp

@@ -19,24 +19,28 @@
 
 namespace ens {
 
-template<typename FunctionType>
-double LineSearch::Optimize(FunctionType& function,
-                            const arma::mat& x1,
-                            arma::mat& x2)
+template<typename FunctionType, typename MatType, typename GradType>
+typename MatType::elem_type LineSearch::Optimize(FunctionType& function,
+                                                 const MatType& x1,
+                                                 MatType& x2)
 {
-  typedef Function<FunctionType> FullFunctionType;
+  typedef typename MatType::elem_type ElemType;
+
+  typedef Function<FunctionType, MatType, GradType> FullFunctionType;
   FullFunctionType& f = static_cast<FullFunctionType&>(function);
 
   // Check that we have all the functions we will need.
-  traits::CheckFunctionTypeAPI<FullFunctionType>();
+  traits::CheckFunctionTypeAPI<FullFunctionType, MatType, GradType>();
 
   // Set up the search line, that is,
   // find the zero of der(gamma) = Derivative(gamma).
-  arma::mat deltaX = x2 - x1;
-  double gamma = 0;
-  double derivative = Derivative(f, x1, deltaX, 0);
-  double derivativeNew = Derivative(f, x1, deltaX, 1);
-  double secant = derivativeNew - derivative;
+  MatType deltaX = x2 - x1;
+  ElemType gamma = 0;
+  ElemType derivative = Derivative<FunctionType, MatType, GradType>(f, x1,
+      deltaX, 0);
+  ElemType derivativeNew = Derivative<FunctionType, MatType, GradType>(f, x1,
+      deltaX, 1);
+  ElemType secant = derivativeNew - derivative;
 
   if (derivative >= 0.0) // Optimal solution at left endpoint.
   {
@@ -65,12 +69,13 @@ double LineSearch::Optimize(FunctionType& function,
     }
 
     // Solve new gamma.
-    double gammaNew = gamma - derivative / secant;
-    gammaNew = std::max(gammaNew, 0.0);
-    gammaNew = std::min(gammaNew, 1.0);
+    ElemType gammaNew = gamma - derivative / secant;
+    gammaNew = std::max(gammaNew, ElemType(0.0));
+    gammaNew = std::min(gammaNew, ElemType(1.0));
 
-    // Update secant, gamma and derivative
-    derivativeNew = Derivative(function, x1, deltaX, gammaNew);
+    // Update secant, gamma and derivative.
+    derivativeNew = Derivative<FunctionType, MatType, GradType>(function, x1,
+        deltaX, gammaNew);
     secant = (derivativeNew - derivative) / (gammaNew - gamma);
     gamma = gammaNew;
     derivative = derivativeNew;
@@ -93,17 +98,17 @@ double LineSearch::Optimize(FunctionType& function,
 
 
 //! Derivative of the function along the search line.
-template<typename FunctionType>
-double LineSearch::Derivative(FunctionType& function,
-                              const arma::mat& x0,
-                              const arma::mat& deltaX,
-                              const double gamma)
+template<typename FunctionType, typename MatType, typename GradType>
+typename MatType::elem_type LineSearch::Derivative(FunctionType& function,
+                                                   const MatType& x0,
+                                                   const MatType& deltaX,
+                                                   const double gamma)
 {
-  arma::mat gradient(x0.n_rows, x0.n_cols);
+  GradType gradient(x0.n_rows, x0.n_cols);
   function.Gradient(x0 + gamma * deltaX, gradient);
   return arma::dot(gradient, deltaX);
 }
 
-
 } // namespace ens
+
 #endif

include/ensmallen_bits/fw/proximal/proximal.hpp

@@ -35,7 +35,8 @@ class Proximal
    *          also saved in v.
    * @param tau Norm of l1 ball.
    */
-  static void ProjectToL1Ball(arma::vec& v, double tau);
+  template<typename MatType>
+  static void ProjectToL1Ball(MatType& v, double tau);
 
   /**
    * Project the vector onto the l0 ball with norm tau. That is, we try to
@@ -48,7 +49,8 @@ class Proximal
    *          also saved in v.
    * @param tau Norm of l0 ball.
    */
-  static void ProjectToL0Ball(arma::vec& v, int tau);
+  template<typename MatType>
+  static void ProjectToL0Ball(MatType& v, int tau);
 };  // class Proximal
 
 } // namespace ens

include/ensmallen_bits/fw/proximal/proximal_impl.hpp

@@ -32,30 +32,31 @@ namespace ens {
  *
  * This is just a soft thresholding.
  */
-inline void Proximal::ProjectToL1Ball(arma::vec& v, double tau)
+template<typename MatType>
+inline void Proximal::ProjectToL1Ball(MatType& v, double tau)
 {
-  arma::vec simplexSol = arma::abs(v);
+  MatType simplexSol = arma::abs(v);
 
   // Already with L1 norm <= tau.
   if (arma::accu(simplexSol) <= tau)
     return;
 
   simplexSol = arma::sort(simplexSol, "descend");
-  arma::vec simplexSum = arma::cumsum(simplexSol);
+  MatType simplexSum = arma::cumsum(simplexSol);
 
   double nu = 0;
-  size_t rho;
+  size_t rho = 0;
   for (size_t j = 1; j <= simplexSol.n_rows; j++)
   {
     rho = simplexSol.n_rows - j;
-    nu = simplexSol(rho) - (simplexSum(rho) - tau)/(rho + 1);
+    nu = simplexSol(rho) - (simplexSum(rho) - tau) / (rho + 1);
     if (nu > 0)
       break;
   }
-  double theta = (simplexSum(rho) - tau)/rho;
+  double theta = (simplexSum(rho) - tau) / rho;
 
   // Threshold on absolute value of v with theta.
-  for (arma::uword j = 0; j< simplexSol.n_rows; j++)
+  for (arma::uword j = 0; j < simplexSol.n_rows; j++)
   {
     if (v(j) >= 0.0)
       v(j) = std::max(v(j) - theta, 0.0);
@@ -68,7 +69,8 @@ inline void Proximal::ProjectToL1Ball(arma::vec& v, double tau)
  * Approximate the vector v with a tau-sparse vector.
  * This is a hard-thresholding.
  */
-inline void Proximal::ProjectToL0Ball(arma::vec& v, int tau)
+template<typename MatType>
+inline void Proximal::ProjectToL0Ball(MatType& v, int tau)
 {
   arma::uvec indices = arma::sort_index(arma::abs(v));
   arma::uword numberToKill = v.n_elem - tau;

include/ensmallen_bits/fw/update_classic.hpp

@@ -36,20 +36,20 @@ class UpdateClassic
    *
    * \f$ x_{k+1} = (1-\gamma)x_k + \gamma s \f$, where \f$ \gamma = 2/(k+2) \f$
    *
-   * @param function function to be optimized, not used in this update rule.
-   * @param oldCoords previous solution coords.
-   * @param s current linear_constr_solution result.
-   * @param newCoords output new solution coords.
-   * @param numIter current iteration number
+   * @param function Function to be optimized, not used in this update rule.
+   * @param oldCoords Previous solution coords.
+   * @param s Current linear_constr_solution result.
+   * @param newCoords Output new solution coords.
+   * @param numIter Current iteration number.
    */
-  template<typename FunctionType>
+  template<typename FunctionType, typename MatType, typename GradType>
   void Update(FunctionType& /* function */,
-              const arma::mat& oldCoords,
-              const arma::mat& s,
-              arma::mat& newCoords,
+              const MatType& oldCoords,
+              const MatType& s,
+              MatType& newCoords,
               const size_t numIter)
   {
-    double gamma = 2.0 / (numIter + 2.0);
+    typename MatType::elem_type gamma = 2.0 / (numIter + 2.0);
     newCoords = (1.0 - gamma) * oldCoords + gamma * s;
   }
 };

include/ensmallen_bits/fw/update_full_correction.hpp

@@ -49,31 +49,36 @@ class UpdateFullCorrection
    * Update rule for FrankWolfe, recalculate the coefficents of of current
    * atoms, while satisfying the norm constraint.
    *
-   * @param function function to be optimized.
-   * @param oldCoords previous solution coords.
-   * @param s current linear_constr_solution result.
-   * @param newCoords new output solution coords.
-   * @param numIter current iteration number.
+   * FuncSqType is an ignored type to match the requirements of the class.
+   *
+   * @param function Function to be optimized.
+   * @param oldCoords Previous solution coords.
+   * @param s Current linear_constr_solution result.
+   * @param newCoords New output solution coords.
+   * @param numIter Current iteration number.
    */
+  template<typename FuncSqType, typename MatType, typename GradType>
   void Update(FuncSq& function,
-              const arma::mat& oldCoords,
-              const arma::mat& s,
-              arma::mat& newCoords,
+              const MatType& oldCoords,
+              const MatType& s,
+              MatType& newCoords,
               const size_t /* numIter */)
   {
     // Line search, with explicit solution here.
-    arma::mat v = tau * s - oldCoords;
-    arma::mat b = function.Vectorb();
-    arma::mat A = function.MatrixA();
-    double gamma = arma::dot(b - A * oldCoords, A * v);
+    MatType v = tau * s - oldCoords;
+    MatType b = function.Vectorb();
+    MatType A = function.MatrixA();
+    typename MatType::elem_type gamma = arma::dot(b - A * oldCoords, A * v);
     gamma = gamma / std::pow(arma::norm(A * v, "fro"), 2);
     gamma = std::min(gamma, 1.0);
     atoms.CurrentCoeffs() = (1.0 - gamma) * atoms.CurrentCoeffs();
-    atoms.AddAtom(s, function, gamma * tau);
+    atoms.AddAtom(arma::mat(s), function, gamma * tau);
 
     // Projected gradient method for enhancement.
     atoms.ProjectedGradientEnhancement(function, tau, stepSize);
-    atoms.RecoverVector(newCoords);
+    arma::mat tmp;
+    atoms.RecoverVector(tmp);
+    newCoords = arma::conv_to<MatType>::from(tmp);
   }
 
  private:

include/ensmallen_bits/fw/update_linesearch.hpp

@@ -40,7 +40,6 @@ class UpdateLineSearch
       tolerance(tolerance), maxIterations(maxIterations)
   {/* Do nothing */}
 
-
   /**
    * Update rule for FrankWolfe, optimize with line search using secant method.
    *
@@ -64,18 +63,19 @@ class UpdateLineSearch
    * @param newCoords output new solution coords.
    * @param numIter current iteration number, not used here.
    */
-  template<typename FunctionType>
+  template<typename FunctionType, typename MatType, typename GradType>
   void Update(FunctionType& function,
-              const arma::mat& oldCoords,
-              const arma::mat& s,
-              arma::mat& newCoords,
+              const MatType& oldCoords,
+              const MatType& s,
+              MatType& newCoords,
               const size_t /* numIter */)
 
   {
     LineSearch solver(maxIterations, tolerance);
 
     newCoords = s;
-    solver.Optimize(function, oldCoords, newCoords);
+    solver.Optimize<FunctionType, MatType, GradType>(function, oldCoords,
+        newCoords);
   }
 
   //! Get the tolerance for termination.

include/ensmallen_bits/fw/update_span.hpp

@@ -45,22 +45,25 @@ class UpdateSpan
    * @param newCoords output new solution coords.
    * @param numIter current iteration number.
    */
+  template<typename FuncSqType, typename MatType, typename GradType>
   void Update(FuncSq& function,
-              const arma::mat& oldCoords,
-              const arma::mat& s,
-              arma::mat& newCoords,
+              const MatType& oldCoords,
+              const MatType& s,
+              MatType& newCoords,
               const size_t /* numIter */)
   {
     // Add new atom into soluton space.
-    atoms.AddAtom(s, function);
+    atoms.AddAtom(arma::mat(s), function);
 
     // Reoptimize the solution in the current space.
     arma::vec b = function.Vectorb();
-    atoms.CurrentCoeffs() = solve(function.MatrixA() * atoms.CurrentAtoms(), b);
+    atoms.CurrentCoeffs() = solve(function.MatrixA() * atoms.CurrentAtoms(), b, arma::solve_opts::fast);
 
     // x has coords of only the current atoms, recover the solution
     // to the original size.
-    atoms.RecoverVector(newCoords);
+    arma::mat tmp;
+    atoms.RecoverVector(tmp);
+    newCoords = arma::conv_to<MatType>::from(tmp);
 
     // Prune the support.
     if (isPrune)
@@ -68,7 +71,8 @@ class UpdateSpan
       double oldF = function.Evaluate(oldCoords);
       double F = 0.25 * oldF + 0.75 * function.Evaluate(newCoords);
       atoms.PruneSupport(F, function);
-      atoms.RecoverVector(newCoords);
+      atoms.RecoverVector(tmp);
+      newCoords = arma::conv_to<MatType>::from(tmp);
     }
   }
 

include/ensmallen_bits/gradient_descent/gradient_descent.hpp

@@ -65,12 +65,36 @@ class GradientDescent
    * the final objective value is returned.
    *
    * @tparam FunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename FunctionType>
-  double Optimize(FunctionType& function, arma::mat& iterate);
+  template<typename FunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(FunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(DecomposableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<DecomposableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
 
   /**
    * Assert all dimensions are numeric and optimize the given function using
@@ -82,19 +106,44 @@ class GradientDescent
    * tuning module.
    *
    * @tparam FunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
    * @param categoricalDimensions A vector of dimension information.  If a value
    *     is true, then that dimension is a categorical dimension.
    * @param numCategories Number of categories in each categorical dimension.
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename FunctionType>
-  double Optimize(
-       FunctionType& function,
-       arma::mat& iterate,
-       const std::vector<bool>& categoricalDimensions,
-       const arma::Row<size_t>& numCategories);
+  template<typename FunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(FunctionType& function,
+           MatType& iterate,
+           const std::vector<bool>& categoricalDimensions,
+           const arma::Row<size_t>& numCategories,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(
+      FunctionType& function,
+      MatType& iterate,
+      const std::vector<bool>& categoricalDimensions,
+      const arma::Row<size_t>& numCategories,
+      CallbackTypes&&... callbacks)
+  {
+    return Optimize<FunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate, categoricalDimensions,
+        numCategories, std::forward<CallbackTypes>(callbacks)...);
+  }
 
   //! Get the step size.
   double StepSize() const { return stepSize; }

include/ensmallen_bits/gradient_descent/gradient_descent_impl.hpp

@@ -30,27 +30,50 @@ inline GradientDescent::GradientDescent(
 { /* Nothing to do. */ }
 
 //! Optimize the function (minimize).
-template<typename FunctionType>
-double GradientDescent::Optimize(
-    FunctionType& function, arma::mat& iterate)
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value,
+typename MatType::elem_type>::type
+GradientDescent::Optimize(FunctionType& function,
+                          MatType& iterateIn,
+                          CallbackTypes&&... callbacks)
 {
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
   // Use the Function<> wrapper type to provide additional functionality.
-  typedef Function<FunctionType> FullFunctionType;
+  typedef Function<FunctionType, BaseMatType, BaseGradType> FullFunctionType;
   FullFunctionType& f(static_cast<FullFunctionType&>(function));
 
   // Make sure we have the methods that we need.
-  traits::CheckFunctionTypeAPI<FullFunctionType>();
+  traits::CheckFunctionTypeAPI<FullFunctionType, BaseMatType, BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
 
   // To keep track of where we are and how things are going.
-  double overallObjective = std::numeric_limits<double>::max();
-  double lastObjective = std::numeric_limits<double>::max();
+  ElemType overallObjective = std::numeric_limits<ElemType>::max();
+  ElemType lastObjective = std::numeric_limits<ElemType>::max();
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+  BaseGradType gradient(iterate.n_rows, iterate.n_cols);
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
 
   // Now iterate!
-  arma::mat gradient(iterate.n_rows, iterate.n_cols);
-  for (size_t i = 1; i != maxIterations; ++i)
+  terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  for (size_t i = 1; i != maxIterations && !terminate; ++i)
   {
     overallObjective = f.EvaluateWithGradient(iterate, gradient);
 
+    terminate |= Callback::EvaluateWithGradient(*this, f, iterate,
+        overallObjective, gradient, callbacks...);
+
     // Output current objective function.
     Info << "Gradient Descent: iteration " << i << ", objective "
         << overallObjective << "." << std::endl;
@@ -60,6 +83,8 @@ double GradientDescent::Optimize(
       Warn << "Gradient Descent: converged to " << overallObjective
           << "; terminating" << " with failure.  Try a smaller step size?"
           << std::endl;
+
+      Callback::EndOptimization(*this, f, iterate, callbacks...);
       return overallObjective;
     }
 
@@ -67,6 +92,8 @@ double GradientDescent::Optimize(
     {
       Info << "Gradient Descent: minimized within tolerance "
           << tolerance << "; " << "terminating optimization." << std::endl;
+
+      Callback::EndOptimization(*this, f, iterate, callbacks...);
       return overallObjective;
     }
 
@@ -75,19 +102,28 @@ double GradientDescent::Optimize(
 
     // And update the iterate.
     iterate -= stepSize * gradient;
+    terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
   }
 
   Info << "Gradient Descent: maximum iterations (" << maxIterations
       << ") reached; " << "terminating optimization." << std::endl;
+
+  Callback::EndOptimization(*this, f, iterate, callbacks...);
   return overallObjective;
 }
 
-template<typename FunctionType>
-double GradientDescent::Optimize(
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value,
+typename MatType::elem_type>::type
+GradientDescent::Optimize(
     FunctionType& function,
-    arma::mat& iterate,
+    MatType& iterate,
     const std::vector<bool>& categoricalDimensions,
-    const arma::Row<size_t>& numCategories)
+    const arma::Row<size_t>& numCategories,
+    CallbackTypes&&... callbacks)
 {
   if (categoricalDimensions.size() != iterate.n_rows)
   {
@@ -118,7 +154,7 @@ double GradientDescent::Optimize(
     }
   }
 
-  return Optimize(function, iterate);
+  return Optimize(function, iterate, callbacks...);
 }
 
 } // namespace ens

include/ensmallen_bits/grid_search/grid_search.hpp

@@ -30,6 +30,8 @@ class GridSearch
    * possible combinations of values for the parameters specified in
    * datasetInfo.
    *
+   * @tparam FunctionType Type of function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
    * @param function Function to optimize.
    * @param bestParameters Variable for storing results.
    * @param categoricalDimensions Set of dimension types.  If a value is true,
@@ -37,10 +39,10 @@ class GridSearch
    * @param numCategories Number of categories in each categorical dimension.
    * @return Objective value of the final point.
    */
-  template<typename FunctionType>
-  double Optimize(
+  template<typename FunctionType, typename MatType>
+  typename MatType::elem_type Optimize(
       FunctionType& function,
-      arma::mat& bestParameters,
+      MatType& bestParameters,
       const std::vector<bool>& categoricalDimensions,
       const arma::Row<size_t>& numCategories);
 
@@ -52,12 +54,12 @@ class GridSearch
    * (parameters) are specified in the first i rows of the currentParameters
    * argument.
    */
-  template<typename FunctionType>
+  template<typename FunctionType, typename MatType>
   void Optimize(
       FunctionType& function,
-      double& bestObjective,
-      arma::mat& bestParameters,
-      arma::vec& currentParameters,
+      typename MatType::elem_type& bestObjective,
+      MatType& bestParameters,
+      MatType& currentParameters,
       const std::vector<bool>& categoricalDimensions,
       const arma::Row<size_t>& numCategories,
       size_t i);

include/ensmallen_bits/grid_search/grid_search_impl.hpp

@@ -17,10 +17,10 @@
 
 namespace ens {
 
-template<typename FunctionType>
-double GridSearch::Optimize(
+template<typename FunctionType, typename MatType>
+typename MatType::elem_type GridSearch::Optimize(
     FunctionType& function,
-    arma::mat& bestParameters,
+    MatType& bestParameters,
     const std::vector<bool>& categoricalDimensions,
     const arma::Row<size_t>& numCategories)
 {
@@ -35,9 +35,12 @@ double GridSearch::Optimize(
     }
   }
 
-  double bestObjective = std::numeric_limits<double>::max();
-  bestParameters = arma::mat(categoricalDimensions.size(), 1);
-  arma::vec currentParameters = arma::vec(categoricalDimensions.size());
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+
+  ElemType bestObjective = std::numeric_limits<ElemType>::max();
+  bestParameters.set_size(categoricalDimensions.size(), 1);
+  MatType currentParameters(categoricalDimensions.size(), 1);
 
   /* Initialize best parameters for the case (very unlikely though) when no set
    * of parameters gives an objective value better than
@@ -51,18 +54,23 @@ double GridSearch::Optimize(
   return bestObjective;
 }
 
-template<typename FunctionType>
+template<typename FunctionType, typename MatType>
 void GridSearch::Optimize(
     FunctionType& function,
-    double& bestObjective,
-    arma::mat& bestParameters,
-    arma::vec& currentParameters,
+    typename MatType::elem_type& bestObjective,
+    MatType& bestParameters,
+    MatType& currentParameters,
     const std::vector<bool>& categoricalDimensions,
     const arma::Row<size_t>& numCategories,
     size_t i)
 {
-  // Make sure we have the methods that we need.
-  traits::CheckNonDifferentiableFunctionTypeAPI<FunctionType>();
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
+  // Make sure we have the methods that we need.  No restrictions on the matrix
+  // type are needed.
+  traits::CheckNonDifferentiableFunctionTypeAPI<FunctionType, BaseMatType>();
 
   if (i < categoricalDimensions.size())
   {
@@ -75,7 +83,7 @@ void GridSearch::Optimize(
   }
   else
   {
-    double objective = function.Evaluate(currentParameters);
+    ElemType objective = function.Evaluate((BaseMatType&) currentParameters);
     if (objective < bestObjective)
     {
       bestObjective = objective;

include/ensmallen_bits/iqn/iqn.hpp

@@ -75,12 +75,36 @@ class IQN
    * objective value is returned.
    *
    * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate);
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(DecomposableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(DecomposableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<DecomposableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
 
   //! Get the step size.
   double StepSize() const { return stepSize; }

include/ensmallen_bits/iqn/iqn_impl.hpp

@@ -32,10 +32,34 @@ inline IQN::IQN(const double stepSize,
 { /* Nothing to do. */ }
 
 //! Optimize the function (minimize).
-template<typename DecomposableFunctionType>
-double IQN::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
+template<typename DecomposableFunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value,
+typename MatType::elem_type>::type
+IQN::Optimize(DecomposableFunctionType& functionIn,
+              MatType& iterateIn,
+              CallbackTypes&&... callbacks)
 {
-  traits::CheckDecomposableFunctionTypeAPI<DecomposableFunctionType>();
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
+  typedef Function<DecomposableFunctionType, BaseMatType, BaseGradType>
+      FullFunctionType;
+  FullFunctionType& function(static_cast<FullFunctionType&>(functionIn));
+
+  // Make sure we have all the methods that we need.
+  traits::CheckDecomposableFunctionTypeAPI<FullFunctionType, BaseMatType,
+      BaseGradType>();
+  RequireDenseFloatingPointType<BaseMatType>();
+  RequireDenseFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
+
+  traits::CheckDecomposableFunctionTypeAPI<DecomposableFunctionType,
+      BaseMatType, BaseGradType>();
 
   // Find the number of functions.
   const size_t numFunctions = function.NumFunctions();
@@ -43,42 +67,54 @@ double IQN::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
   if (numFunctions % batchSize != 0)
     ++numBatches; // Capture last few.
 
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
   // To keep track of where we are and how things are going.
-  double overallObjective = 0;
+  ElemType overallObjective = 0;
 
-  arma::cube y(iterate.n_rows, iterate.n_cols, numBatches);
-  arma::cube t(iterate.n_elem, 1, numBatches);
-  arma::cube Q(iterate.n_elem, iterate.n_elem, numBatches);
-  arma::mat initialIterate = arma::randn(iterate.n_rows, iterate.n_cols);
-  arma::mat B = arma::eye(iterate.n_elem, iterate.n_elem);
+  // Controls early termination of the optimization process.
+  bool terminate = false;
 
-  arma::mat g = arma::zeros(iterate.n_rows, iterate.n_cols);
+  std::vector<BaseGradType> y(numBatches, BaseGradType(iterate.n_rows,
+      iterate.n_cols));
+  std::vector<BaseMatType> t(numBatches, BaseMatType(iterate.n_rows,
+      iterate.n_cols));
+  std::vector<BaseMatType> Q(numBatches, BaseMatType(iterate.n_elem,
+      iterate.n_elem));
+  BaseMatType initialIterate = arma::randn<arma::Mat<ElemType>>(iterate.n_rows,
+      iterate.n_cols);
+  BaseGradType B(iterate.n_elem, iterate.n_elem);
+  B.eye();
+
+  BaseGradType g(iterate.n_rows, iterate.n_cols);
+  g.zeros();
   for (size_t i = 0, f = 0; i < numFunctions; f++)
   {
     // Find the effective batch size (the last batch may be smaller).
     const size_t effectiveBatchSize = std::min(batchSize, numFunctions - i);
 
-    t.slice(f) = arma::mat(initialIterate.memptr(), iterate.n_elem,
-        1, false, false);
-    function.Gradient(initialIterate, i, y.slice(f), effectiveBatchSize);
+    // It would be nice to avoid this copy but it is difficult to be generic to
+    // any MatType and still do that.
+    t[f] = initialIterate;
+    function.Gradient(initialIterate, i, y[f], effectiveBatchSize);
 
-    Q.slice(f).eye();
-    g += y.slice(f);
-    y.slice(f) /= (double) effectiveBatchSize;
+    terminate |= Callback::Gradient(*this, function, initialIterate,
+        y[f], callbacks...);
+
+    Q[f].eye();
+    g += y[f];
+    y[f] /= (double) effectiveBatchSize;
 
     i += effectiveBatchSize;
   }
   g /= numFunctions;
 
-  arma::mat gradient(iterate.n_rows, iterate.n_cols);
-  arma::mat u = t.slice(0);
+  BaseGradType gradient(iterate.n_rows, iterate.n_cols);
+  BaseMatType u = t[0];
 
-  // Convenience alias to avoid multiple use of arma::vectorise.
-  arma::mat iterateVec = arma::mat(iterate.memptr(), iterate.n_elem,
-      1, false, false);
-  arma::mat gVec = arma::mat(g.memptr(), iterate.n_elem, 1, false, false);
-
-  for (size_t i = 1; i != maxIterations; ++i)
+  terminate |= Callback::BeginOptimization(*this, function, iterate,
+      callbacks...);
+  for (size_t i = 1; i != maxIterations && !terminate; ++i)
   {
     for (size_t j = 0, f = 0; f < numFunctions; j++)
     {
@@ -89,36 +125,43 @@ double IQN::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
       const size_t effectiveBatchSize = std::min(batchSize, numFunctions -
           it * batchSize);
 
-      if (arma::norm(iterateVec - t.slice(it)) > 0)
+      if (arma::norm(iterate - t[it]) > 0)
       {
         function.Gradient(iterate, it * batchSize, gradient,
             effectiveBatchSize);
         gradient /= effectiveBatchSize;
 
-        const arma::mat s = iterateVec - t.slice(it);
-        const arma::mat yy = arma::vectorise(gradient - y.slice(it));
+        terminate |= Callback::Gradient(*this, function, iterate, gradient,
+            callbacks...);
 
-        const arma::mat stochasticHessian = Q.slice(it) + yy * yy.t() /
-            arma::as_scalar(yy.t() * s) - Q.slice(it) * s * s.t() *
-            Q.slice(it) / arma::as_scalar(s.t() * Q.slice(it) * s);
+        const BaseMatType s = arma::vectorise(iterate - t[it]);
+        const BaseGradType yy = arma::vectorise(gradient - y[it]);
+
+        const BaseGradType stochasticHessian = Q[it] + yy * yy.t() /
+            arma::as_scalar(yy.t() * s) - Q[it] * s * s.t() *
+            Q[it] / arma::as_scalar(s.t() * Q[it] * s);
 
         // Update aggregate Hessian approximation.
-        B += (1.0 / numBatches) * (stochasticHessian - Q.slice(it));
+        B += (1.0 / numBatches) * (stochasticHessian - Q[it]);
 
         // Update aggregate Hessian-variable product.
-        u += (1.0 / numBatches) * (stochasticHessian * iterateVec -
-            Q.slice(it) * t.slice(it));
+        u += arma::reshape((1.0 / numBatches) * (stochasticHessian *
+            arma::vectorise(iterate) - Q[it] * arma::vectorise(t[it])),
+            u.n_rows, u.n_cols);;
 
         // Update aggregate gradient.
-        g += (1.0 / numBatches) * (gradient - y.slice(it));
+        g += (1.0 / numBatches) * (gradient - y[it]);
 
         // Update the function information tables.
-        Q.slice(it) = stochasticHessian;
-        y.slice(it) = gradient;
-        t.slice(it) = iterateVec;
+        Q[it] = std::move(stochasticHessian);
+        y[it] = std::move(gradient);
+        t[it] = iterate;
 
-        iterateVec = stepSize * B.i() * (u - gVec) + (1 - stepSize) *
-            iterateVec;
+        iterate = arma::reshape(stepSize * B.i() * (u.t() - arma::vectorise(g)),
+            iterate.n_rows, iterate.n_cols) + (1 - stepSize) * iterate;
+
+        terminate |= Callback::StepTaken(*this, function, iterate,
+            callbacks...);
       }
 
       f += effectiveBatchSize;
@@ -128,7 +171,12 @@ double IQN::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
     for (size_t f = 0; f < numFunctions; f += batchSize)
     {
       const size_t effectiveBatchSize = std::min(batchSize, numFunctions - f);
-      overallObjective += function.Evaluate(iterate, f, effectiveBatchSize);
+      const ElemType objective = function.Evaluate(iterate, f,
+          effectiveBatchSize);
+      overallObjective += objective;
+
+      Callback::Evaluate(*this, function, iterate, objective,
+          callbacks...);
     }
     overallObjective /= numFunctions;
 
@@ -140,6 +188,8 @@ double IQN::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
     {
       Warn << "IQN: converged to " << overallObjective << "; terminating"
           << " with failure.  Try a smaller step size?" << std::endl;
+
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
       return overallObjective;
     }
 
@@ -147,6 +197,8 @@ double IQN::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
     {
       Info << "IQN: minimized within tolerance " << tolerance << "; "
           << "terminating optimization." << std::endl;
+
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
       return overallObjective;
     }
   }
@@ -154,6 +206,7 @@ double IQN::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
   Info << "IQN: maximum iterations (" << maxIterations << ") reached; "
       << "terminating optimization." << std::endl;
 
+  Callback::EndOptimization(*this, function, iterate, callbacks...);
   return overallObjective;
 }
 

include/ensmallen_bits/katyusha/katyusha.hpp

@@ -63,6 +63,8 @@ class KatyushaType
    * @param tolerance Maximum absolute tolerance to terminate algorithm.
    * @param shuffle If true, the function order is shuffled; otherwise, each
    *    function is visited in linear order.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
    */
   KatyushaType(const double convexity = 1.0,
                const double lipschitz = 10.0,
@@ -70,7 +72,8 @@ class KatyushaType
                const size_t maxIterations = 1000,
                const size_t innerIterations = 0,
                const double tolerance = 1e-5,
-               const bool shuffle = true);
+               const bool shuffle = true,
+               const bool exactObjective = false);
 
   /**
    * Optimize the given function using Katyusha. The given starting point will
@@ -78,12 +81,36 @@ class KatyushaType
    * objective value is returned.
    *
    * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate);
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(DecomposableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(DecomposableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<DecomposableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
 
   //! Get the convexity parameter.
   double Convexity() const { return convexity; }
@@ -120,6 +147,11 @@ class KatyushaType
   //! Modify whether or not the individual functions are shuffled.
   bool& Shuffle() { return shuffle; }
 
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return exactObjective; }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return exactObjective; }
+
  private:
   //! The convexity regularization term.
   double convexity;
@@ -142,6 +174,9 @@ class KatyushaType
   //! Controls whether or not the individual functions are shuffled when
   //! iterating.
   bool shuffle;
+
+  //! Controls whether or not the actual Objective value is calculated.
+  bool exactObjective;
 };
 
 // Convenience typedefs.

include/ensmallen_bits/katyusha/katyusha_impl.hpp

@@ -27,24 +27,46 @@ KatyushaType<Proximal>::KatyushaType(
     const size_t maxIterations,
     const size_t innerIterations,
     const double tolerance,
-    const bool shuffle) :
+    const bool shuffle,
+    const bool exactObjective) :
     convexity(convexity),
     lipschitz(lipschitz),
     batchSize(batchSize),
     maxIterations(maxIterations),
     innerIterations(innerIterations),
     tolerance(tolerance),
-    shuffle(shuffle)
+    shuffle(shuffle),
+    exactObjective(exactObjective)
 { /* Nothing to do. */ }
 
 //! Optimize the function (minimize).
 template<bool Proximal>
-template<typename DecomposableFunctionType>
-double KatyushaType<Proximal>::Optimize(
+template<typename DecomposableFunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value,
+typename MatType::elem_type>::type
+KatyushaType<Proximal>::Optimize(
     DecomposableFunctionType& function,
-    arma::mat& iterate)
+    MatType& iterateIn,
+    CallbackTypes&&... callbacks)
 {
-  traits::CheckDecomposableFunctionTypeAPI<DecomposableFunctionType>();
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
+  traits::CheckDecomposableFunctionTypeAPI<DecomposableFunctionType,
+      BaseMatType, BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
 
   // Find the number of functions to use.
   const size_t numFunctions = function.NumFunctions();
@@ -74,29 +96,36 @@ double KatyushaType<Proximal>::Optimize(
   normalizer = 1.0 / normalizer;
 
   // To keep track of where we are and how things are going.
-  double overallObjective = 0;
-  double lastObjective = DBL_MAX;
+  ElemType overallObjective = 0;
+  ElemType lastObjective = DBL_MAX;
 
   // Now iterate!
-  arma::mat gradient(iterate.n_rows, iterate.n_cols);
-  arma::mat fullGradient(iterate.n_rows, iterate.n_cols);
-  arma::mat gradient0(iterate.n_rows, iterate.n_cols);
+  BaseGradType gradient(iterate.n_rows, iterate.n_cols);
+  BaseGradType fullGradient(iterate.n_rows, iterate.n_cols);
+  BaseGradType gradient0(iterate.n_rows, iterate.n_cols);
 
-  arma::mat iterate0 = iterate;
-  arma::mat y = iterate;
-  arma::mat z = iterate;
-  arma::mat w = arma::zeros<arma::mat>(iterate.n_rows, iterate.n_cols);
+  BaseMatType iterate0 = iterate;
+  BaseMatType y = iterate;
+  BaseMatType z = iterate;
+  BaseMatType w(iterate.n_rows, iterate.n_cols);
+  w.zeros();
 
   const size_t actualMaxIterations = (maxIterations == 0) ?
       std::numeric_limits<size_t>::max() : maxIterations;
-  for (size_t i = 0; i < actualMaxIterations; ++i)
+  terminate |= Callback::BeginOptimization(*this, function, iterate,
+      callbacks...);
+  for (size_t i = 0; i < actualMaxIterations && !terminate; ++i)
   {
     // Calculate the objective function.
     overallObjective = 0;
     for (size_t f = 0; f < numFunctions; f += batchSize)
     {
       const size_t effectiveBatchSize = std::min(batchSize, numFunctions - f);
-      overallObjective += function.Evaluate(iterate0, f, effectiveBatchSize);
+      const ElemType objective = function.Evaluate(iterate0, f,
+          effectiveBatchSize);
+      overallObjective += objective;
+
+      Callback::Evaluate(*this, function, iterate0, objective, callbacks...);
     }
 
     if (std::isnan(overallObjective) || std::isinf(overallObjective))
@@ -104,6 +133,8 @@ double KatyushaType<Proximal>::Optimize(
       Warn << "Katyusha: converged to " << overallObjective
           << "; terminating  with failure.  Try a smaller step size?"
           << std::endl;
+
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
       return overallObjective;
     }
 
@@ -111,6 +142,8 @@ double KatyushaType<Proximal>::Optimize(
     {
       Info << "Katyusha: minimized within tolerance " << tolerance
           << "; terminating optimization." << std::endl;
+
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
       return overallObjective;
     }
 
@@ -119,6 +152,8 @@ double KatyushaType<Proximal>::Optimize(
     // Compute the full gradient.
     size_t effectiveBatchSize = std::min(batchSize, numFunctions);
     function.Gradient(iterate, 0, fullGradient, effectiveBatchSize);
+    terminate |= Callback::Gradient(*this, function, iterate, fullGradient,
+          callbacks...);
     for (size_t f = effectiveBatchSize; f < numFunctions;
         /* incrementing done manually */)
     {
@@ -128,6 +163,9 @@ double KatyushaType<Proximal>::Optimize(
       function.Gradient(iterate0, f, gradient, effectiveBatchSize);
       fullGradient += gradient;
 
+      terminate |= Callback::Gradient(*this, function, iterate0, gradient,
+          callbacks...);
+
       f += effectiveBatchSize;
     }
     fullGradient /= (double) numFunctions;
@@ -153,15 +191,23 @@ double KatyushaType<Proximal>::Optimize(
       effectiveBatchSize = std::min(batchSize, numFunctions - currentFunction);
       iterate = tau1 * z + tau2 * iterate0 + (1 - tau1 - tau2) * y;
 
+      terminate |= Callback::StepTaken(*this, function, iterate,
+            callbacks...);
+
       // Calculate variance reduced gradient.
       function.Gradient(iterate, currentFunction, gradient,
           effectiveBatchSize);
+      terminate |= Callback::Gradient(*this, function, iterate, gradient,
+          callbacks...);
+
       function.Gradient(iterate0, currentFunction, gradient0,
           effectiveBatchSize);
+      terminate |= Callback::Gradient(*this, function, iterate0, gradient0,
+          callbacks...);
 
       // By the minimality definition of z_{k + 1}, we have that:
       // z_{k+1} − z_k + \alpha * \sigma_{k+1} + \alpha g = 0.
-      arma::mat zNew = z - alpha * (fullGradient + (gradient - gradient0) /
+      BaseMatType zNew = z - alpha * (fullGradient + (gradient - gradient0) /
           (double) batchSize);
 
       // Proximal update, choose between Option I and Option II. Shift relative
@@ -197,13 +243,22 @@ double KatyushaType<Proximal>::Optimize(
   Info << "Katyusha: maximum iterations (" << maxIterations << ") reached"
       << "; terminating optimization." << std::endl;
 
-  // Calculate final objective.
-  overallObjective = 0;
-  for (size_t i = 0; i < numFunctions; i += batchSize)
+  // Calculate final objective if exactObjective is set to true.
+  if (exactObjective)
   {
-    const size_t effectiveBatchSize = std::min(batchSize, numFunctions - i);
-    overallObjective += function.Evaluate(iterate, i, effectiveBatchSize);
+    overallObjective = 0;
+    for (size_t i = 0; i < numFunctions; i += batchSize)
+    {
+      const size_t effectiveBatchSize = std::min(batchSize, numFunctions - i);
+      const ElemType objective = function.Evaluate(iterate, i,
+          effectiveBatchSize);
+      overallObjective += objective;
+
+      Callback::Evaluate(*this, function, iterate, objective, callbacks...);
+    }
   }
+
+  Callback::EndOptimization(*this, function, iterate, callbacks...);
   return overallObjective;
 }
 

include/ensmallen_bits/lbfgs/lbfgs.hpp

@@ -59,14 +59,6 @@ class L_BFGS
          const double minStep = 1e-20,
          const double maxStep = 1e20);
 
-  /**
-   * Return the point where the lowest function value has been found.
-   *
-   * @return arma::vec representing the point and a double with the function
-   *     value at that point.
-   */
-  const std::pair<arma::mat, double>& MinPointIterate() const;
-
   /**
    * Use L-BFGS to optimize the given function, starting at the given iterate
    * point and finding the minimum.  The maximum number of iterations is set in
@@ -76,12 +68,36 @@ class L_BFGS
    * algorithm, and the final objective value is returned.
    *
    * @tparam FunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize; must have Evaluate() and Gradient().
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename FunctionType>
-  double Optimize(FunctionType& function, arma::mat& iterate);
+  template<typename FunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(FunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(DecomposableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<DecomposableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
 
   //! Get the memory size.
   size_t NumBasis() const { return numBasis; }
@@ -147,6 +163,8 @@ class L_BFGS
   double minStep;
   //! Maximum step of the line search.
   double maxStep;
+  //! Controls early termination of the optimization process.
+  bool terminate;
 
   /**
    * Calculate the scaling factor, gamma, which is used to scale the Hessian
@@ -158,10 +176,11 @@ class L_BFGS
    * @param s Differences between the iterate and old iterate matrix.
    * @param y Differences between the gradient and the old gradient matrix.
    */
+  template<typename MatType, typename CubeType>
   double ChooseScalingFactor(const size_t iterationNum,
-                             const arma::mat& gradient,
-                             const arma::cube& s,
-                             const arma::cube& y);
+                             const MatType& gradient,
+                             const CubeType& s,
+                             const CubeType& y);
 
   /**
    * Perform a back-tracking line search along the search direction to
@@ -173,17 +192,24 @@ class L_BFGS
    * @param iterate The initial point to begin the line search from.
    * @param gradient The gradient at the initial point.
    * @param searchDirection A vector specifying the search direction.
-   * @param stepSize Variable the calculated step size will be stored in.
+   * @param finalStepSize The resulting step size (0 if no step).
+   * @param callbacks Callback functions.
    *
    * @return false if no step size is suitable, true otherwise.
    */
-  template<typename FunctionType>
+  template<typename FunctionType,
+           typename ElemType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
   bool LineSearch(FunctionType& function,
-                  double& functionValue,
-                  arma::mat& iterate,
-                  arma::mat& gradient,
-                  arma::mat& newIterateTmp,
-                  const arma::mat& searchDirection);
+                  ElemType& functionValue,
+                  MatType& iterate,
+                  GradType& gradient,
+                  MatType& newIterateTmp,
+                  const GradType& searchDirection,
+                  double& finalStepSize,
+                  CallbackTypes&... callbacks);
 
   /**
    * Find the L-BFGS search direction.
@@ -195,12 +221,13 @@ class L_BFGS
    * @param y Differences between the gradient and the old gradient matrix.
    * @param searchDirection Vector to store search direction in.
    */
-  void SearchDirection(const arma::mat& gradient,
+  template<typename MatType, typename CubeType>
+  void SearchDirection(const MatType& gradient,
                        const size_t iterationNum,
                        const double scalingFactor,
-                       const arma::cube& s,
-                       const arma::cube& y,
-                       arma::mat& searchDirection);
+                       const CubeType& s,
+                       const CubeType& y,
+                       MatType& searchDirection);
 
   /**
    * Update the y and s matrices, which store the differences
@@ -215,13 +242,14 @@ class L_BFGS
    * @param s Differences between the iterate and old iterate matrix.
    * @param y Differences between the gradient and the old gradient matrix.
    */
+  template<typename MatType, typename GradType, typename CubeType>
   void UpdateBasisSet(const size_t iterationNum,
-                      const arma::mat& iterate,
-                      const arma::mat& oldIterate,
-                      const arma::mat& gradient,
-                      const arma::mat& oldGradient,
-                      arma::cube& s,
-                      arma::cube& y);
+                      const MatType& iterate,
+                      const MatType& oldIterate,
+                      const GradType& gradient,
+                      const GradType& oldGradient,
+                      CubeType& s,
+                      CubeType& y);
 };
 
 } // namespace ens

include/ensmallen_bits/lbfgs/lbfgs_impl.hpp

@@ -55,7 +55,8 @@ inline L_BFGS::L_BFGS(const size_t numBasis,
     factr(factr),
     maxLineSearchTrials(maxLineSearchTrials),
     minStep(minStep),
-    maxStep(maxStep)
+    maxStep(maxStep),
+    terminate(false)
 {
   // Nothing to do.
 }
@@ -70,18 +71,21 @@ inline L_BFGS::L_BFGS(const size_t numBasis,
  * @param s Differences between the iterate and old iterate matrix.
  * @param y Differences between the gradient and the old gradient matrix.
  */
-inline double L_BFGS::ChooseScalingFactor(const size_t iterationNum,
-                                          const arma::mat& gradient,
-                                          const arma::cube& s,
-                                          const arma::cube& y)
+template<typename MatType, typename CubeType>
+double L_BFGS::ChooseScalingFactor(const size_t iterationNum,
+                                   const MatType& gradient,
+                                   const CubeType& s,
+                                   const CubeType& y)
 {
+  typedef typename CubeType::elem_type CubeElemType;
+
   double scalingFactor = 1.0;
   if (iterationNum > 0)
   {
     int previousPos = (iterationNum - 1) % numBasis;
     // Get s and y matrices once instead of multiple times.
-    const arma::mat& sMat = s.slice(previousPos);
-    const arma::mat& yMat = y.slice(previousPos);
+    const arma::Mat<CubeElemType>& sMat = s.slice(previousPos);
+    const arma::Mat<CubeElemType>& yMat = y.slice(previousPos);
     scalingFactor = dot(sMat, yMat) / dot(yMat, yMat);
   }
   else
@@ -102,22 +106,24 @@ inline double L_BFGS::ChooseScalingFactor(const size_t iterationNum,
  * @param y Differences between the gradient and the old gradient matrix.
  * @param searchDirection Vector to store search direction in.
  */
-inline void L_BFGS::SearchDirection(const arma::mat& gradient,
-                                    const size_t iterationNum,
-                                    const double scalingFactor,
-                                    const arma::cube& s,
-                                    const arma::cube& y,
-                                    arma::mat& searchDirection)
+template<typename MatType, typename CubeType>
+void L_BFGS::SearchDirection(const MatType& gradient,
+                             const size_t iterationNum,
+                             const double scalingFactor,
+                             const CubeType& s,
+                             const CubeType& y,
+                             MatType& searchDirection)
 {
   // Start from this point.
   searchDirection = gradient;
 
   // See "A Recursive Formula to Compute H * g" in "Updating quasi-Newton
   // matrices with limited storage" (Nocedal, 1980).
+  typedef typename CubeType::elem_type CubeElemType;
 
   // Temporary variables.
-  arma::vec rho(numBasis);
-  arma::vec alpha(numBasis);
+  arma::Col<CubeElemType> rho(numBasis);
+  arma::Col<CubeElemType> alpha(numBasis);
 
   size_t limit = (numBasis > iterationNum) ? 0 : (iterationNum - numBasis);
   for (size_t i = iterationNum; i != limit; i--)
@@ -158,13 +164,14 @@ inline void L_BFGS::SearchDirection(const arma::mat& gradient,
  * @param s Differences between the iterate and old iterate matrix.
  * @param y Differences between the gradient and the old gradient matrix.
  */
-inline void L_BFGS::UpdateBasisSet(const size_t iterationNum,
-                                   const arma::mat& iterate,
-                                   const arma::mat& oldIterate,
-                                   const arma::mat& gradient,
-                                   const arma::mat& oldGradient,
-                                   arma::cube& s,
-                                   arma::cube& y)
+template<typename MatType, typename GradType, typename CubeType>
+void L_BFGS::UpdateBasisSet(const size_t iterationNum,
+                            const MatType& iterate,
+                            const MatType& oldIterate,
+                            const GradType& gradient,
+                            const GradType& oldGradient,
+                            CubeType& s,
+                            CubeType& y)
 {
   // Overwrite a certain position instead of pushing everything in the vector
   // back one position.
@@ -182,24 +189,32 @@ inline void L_BFGS::UpdateBasisSet(const size_t iterationNum,
  * @param iterate The initial point to begin the line search from.
  * @param gradient The gradient at the initial point.
  * @param searchDirection A vector specifying the search direction.
- * @param stepSize Variable the calculated step size will be stored in.
+ * @param finalStepSize The resulting step size used.
+ * @param callbacks Callback functions.
  *
  * @return false if no step size is suitable, true otherwise.
  */
-template<typename FunctionType>
+template<typename FunctionType,
+         typename ElemType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
 bool L_BFGS::LineSearch(FunctionType& function,
-                        double& functionValue,
-                        arma::mat& iterate,
-                        arma::mat& gradient,
-                        arma::mat& newIterateTmp,
-                        const arma::mat& searchDirection)
+                        ElemType& functionValue,
+                        MatType& iterate,
+                        GradType& gradient,
+                        MatType& newIterateTmp,
+                        const GradType& searchDirection,
+                        double& finalStepSize,
+                        CallbackTypes&... callbacks)
 {
   // Default first step size of 1.0.
   double stepSize = 1.0;
+  finalStepSize = 0.0; // Set only when we take the step.
 
   // The initial linear term approximation in the direction of the
   // search direction.
-  double initialSearchDirectionDotGradient =
+  ElemType initialSearchDirectionDotGradient =
       arma::dot(gradient, searchDirection);
 
   // If it is not a descent direction, just report failure.
@@ -211,10 +226,10 @@ bool L_BFGS::LineSearch(FunctionType& function,
   }
 
   // Save the initial function value.
-  double initialFunctionValue = functionValue;
+  ElemType initialFunctionValue = functionValue;
 
   // Unit linear approximation to the decrease in function value.
-  double linearApproxFunctionValueDecrease = armijoConstant *
+  ElemType linearApproxFunctionValueDecrease = armijoConstant *
       initialSearchDirectionDotGradient;
 
   // The number of iteration in the search.
@@ -225,7 +240,7 @@ bool L_BFGS::LineSearch(FunctionType& function,
   const double dec = 0.5;
   double width = 0;
   double bestStepSize = 1.0;
-  double bestObjective = std::numeric_limits<double>::max();
+  ElemType bestObjective = std::numeric_limits<ElemType>::max();
 
   while (true)
   {
@@ -234,6 +249,10 @@ bool L_BFGS::LineSearch(FunctionType& function,
     newIterateTmp = iterate;
     newIterateTmp += stepSize * searchDirection;
     functionValue = function.EvaluateWithGradient(newIterateTmp, gradient);
+
+    terminate |= Callback::EvaluateWithGradient(*this, function, newIterateTmp,
+        functionValue, gradient, callbacks...);
+
     if (functionValue < bestObjective)
     {
       bestStepSize = stepSize;
@@ -249,7 +268,8 @@ bool L_BFGS::LineSearch(FunctionType& function,
     else
     {
       // Check Wolfe's condition.
-      double searchDirectionDotGradient = arma::dot(gradient, searchDirection);
+      ElemType searchDirectionDotGradient = arma::dot(gradient,
+          searchDirection);
 
       if (searchDirectionDotGradient < wolfe *
           initialSearchDirectionDotGradient)
@@ -284,6 +304,7 @@ bool L_BFGS::LineSearch(FunctionType& function,
 
   // Move to the new iterate.
   iterate += bestStepSize * searchDirection;
+  finalStepSize = bestStepSize;
   return true;
 }
 
@@ -295,48 +316,74 @@ bool L_BFGS::LineSearch(FunctionType& function,
  *
  * @param numIterations Maximum number of iterations to perform
  * @param iterate Starting point (will be modified)
+ * @param callbacks Callback functions.
  */
-template<typename FunctionType>
-double L_BFGS::Optimize(FunctionType& function, arma::mat& iterate)
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value,
+typename MatType::elem_type>::type
+L_BFGS::Optimize(FunctionType& function,
+                 MatType& iterateIn,
+                 CallbackTypes&&... callbacks)
 {
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
   // Use the Function<> wrapper to ensure the function has all of the functions
   // that we need.
-  typedef Function<FunctionType> FullFunctionType;
+  typedef Function<FunctionType, BaseMatType, BaseGradType> FullFunctionType;
   FullFunctionType& f = static_cast<FullFunctionType&>(function);
 
   // Check that we have all the functions we will need.
-  traits::CheckFunctionTypeAPI<FullFunctionType>();
+  traits::CheckFunctionTypeAPI<FullFunctionType, BaseMatType, BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
 
   // Ensure that the cubes holding past iterations' information are the right
   // size.  Also set the current best point value to the maximum.
   const size_t rows = iterate.n_rows;
   const size_t cols = iterate.n_cols;
 
-  arma::mat newIterateTmp(rows, cols);
-  arma::cube s(rows, cols, numBasis);
-  arma::cube y(rows, cols, numBasis);
+  BaseMatType newIterateTmp(rows, cols);
+  arma::Cube<ElemType> s(rows, cols, numBasis);
+  arma::Cube<ElemType> y(rows, cols, numBasis);
 
   // The old iterate to be saved.
-  arma::mat oldIterate;
-  oldIterate.zeros(iterate.n_rows, iterate.n_cols);
+  BaseMatType oldIterate(iterate.n_rows, iterate.n_cols);
+  oldIterate.zeros();
 
   // Whether to optimize until convergence.
   bool optimizeUntilConvergence = (maxIterations == 0);
 
   // The gradient: the current and the old.
-  arma::mat gradient(iterate.n_rows, iterate.n_cols, arma::fill::zeros);
-  arma::mat oldGradient(iterate.n_rows, iterate.n_cols, arma::fill::zeros);
+  BaseGradType gradient(iterate.n_rows, iterate.n_cols);
+  gradient.zeros();
+  BaseGradType oldGradient(iterate.n_rows, iterate.n_cols);
+  oldGradient.zeros();
 
   // The search direction.
-  arma::mat searchDirection(iterate.n_rows, iterate.n_cols, arma::fill::zeros);
+  BaseGradType searchDirection(iterate.n_rows, iterate.n_cols);
+  searchDirection.zeros();
 
   // The initial function value and gradient.
-  double functionValue = f.EvaluateWithGradient(iterate, gradient);
-  double prevFunctionValue = functionValue;
+  ElemType functionValue = f.EvaluateWithGradient(iterate, gradient);
+
+  terminate |= Callback::EvaluateWithGradient(*this, f, iterate,
+        functionValue, gradient, callbacks...);
+
+  ElemType prevFunctionValue = functionValue;
 
   // The main optimization loop.
-  for (size_t itNum = 0; optimizeUntilConvergence || (itNum != maxIterations);
-       ++itNum)
+  terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  for (size_t itNum = 0; (optimizeUntilConvergence || (itNum != maxIterations))
+      && !terminate; ++itNum)
   {
     prevFunctionValue = functionValue;
 
@@ -346,7 +393,7 @@ double L_BFGS::Optimize(FunctionType& function, arma::mat& iterate)
     // least one descent step.
     if (itNum > 0 && (arma::norm(gradient, 2) < minGradientNorm))
     {
-      Warn << "L-BFGS gradient norm too small (terminating successfully)."
+      Info << "L-BFGS gradient norm too small (terminating successfully)."
           << std::endl;
       break;
     }
@@ -371,8 +418,9 @@ double L_BFGS::Optimize(FunctionType& function, arma::mat& iterate)
     oldIterate = iterate;
     oldGradient = gradient;
 
+    double stepSize; // Set by LineSearch().
     if (!LineSearch(f, functionValue, iterate, gradient, newIterateTmp,
-        searchDirection))
+        searchDirection, stepSize, callbacks...))
     {
       Warn << "Line search failed.  Stopping optimization." << std::endl;
       break; // The line search failed; nothing else to try.
@@ -380,7 +428,7 @@ double L_BFGS::Optimize(FunctionType& function, arma::mat& iterate)
 
     // It is possible that the difference between the two coordinates is zero.
     // In this case we terminate successfully.
-    if (accu(iterate != oldIterate) == 0)
+    if (stepSize == 0.0)
     {
       Info << "L-BFGS step size of 0 (terminating successfully)."
           << std::endl;
@@ -390,7 +438,8 @@ double L_BFGS::Optimize(FunctionType& function, arma::mat& iterate)
     // If we can't make progress on the gradient, then we'll also accept
     // a stable function value.
     const double denom = std::max(
-        std::max(fabs(prevFunctionValue), fabs(functionValue)), 1.0);
+        std::max(std::abs(prevFunctionValue), std::abs(functionValue)),
+        (ElemType) 1.0);
     if ((prevFunctionValue - functionValue) / denom <= factr)
     {
       Info << "L-BFGS function value stable (terminating successfully)."
@@ -400,8 +449,11 @@ double L_BFGS::Optimize(FunctionType& function, arma::mat& iterate)
 
     // Overwrite an old basis set.
     UpdateBasisSet(itNum, iterate, oldIterate, gradient, oldGradient, s, y);
+
+    terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
   } // End of the optimization loop.
 
+  Callback::EndOptimization(*this, f, iterate, callbacks...);
   return functionValue;
 }
 

include/ensmallen_bits/padam/padam.hpp

@@ -64,6 +64,8 @@ class Padam
    *        function is visited in linear order.
    * @param resetPolicy If true, parameters are reset before every Optimize
    *        call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
    */
   Padam(const double stepSize = 0.001,
         const size_t batchSize = 32,
@@ -74,7 +76,8 @@ class Padam
         const size_t maxIterations = 100000,
         const double tolerance = 1e-5,
         const bool shuffle = true,
-        const bool resetPolicy = true) :
+        const bool resetPolicy = true,
+        const bool exactObjective = false) :
       optimizer(stepSize,
                 batchSize,
                 maxIterations,
@@ -82,7 +85,8 @@ class Padam
                 shuffle,
                 PadamUpdate(epsilon, beta1, beta2, partial),
                 NoDecay(),
-                resetPolicy)
+                resetPolicy,
+                exactObjective)
   { /* Nothing to do here. */ }
 
   /**
@@ -91,14 +95,38 @@ class Padam
    * objective value is returned.
    *
    * @tparam DecomposableFunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate)
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(DecomposableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
   {
-    return optimizer.Optimize(function, iterate);
+    return optimizer.template Optimize<
+        DecomposableFunctionType, MatType, GradType, CallbackTypes...>(
+        function, iterate, callbacks...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(DecomposableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<DecomposableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
   }
 
   //! Get the step size.
@@ -146,6 +174,11 @@ class Padam
   //! Modify whether or not the individual functions are shuffled.
   bool& Shuffle() { return optimizer.Shuffle(); }
 
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
   //! Get whether or not the update policy parameters
   //! are reset before Optimize call.
   bool ResetPolicy() const { return optimizer.ResetPolicy(); }

include/ensmallen_bits/padam/padam_update.hpp

@@ -56,51 +56,6 @@ class PadamUpdate
     // Nothing to do.
   }
 
-  /**
-   * The Initialize method is called by SGD Optimizer method before the start of
-   * the iteration update process.
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    m = arma::zeros<arma::mat>(rows, cols);
-    v = arma::zeros<arma::mat>(rows, cols);
-    vImproved = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for Padam.
-   *
-   * @param iterate Parameters that minimize the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    // Increment the iteration counter variable.
-    ++iteration;
-
-    // And update the iterate.
-    m *= beta1;
-    m += (1 - beta1) * gradient;
-
-    v *= beta2;
-    v += (1 - beta2) * (gradient % gradient);
-
-    const double biasCorrection1 = 1.0 - std::pow(beta1, iteration);
-    const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
-
-    // Element wise maximum of past and present squared gradients.
-    vImproved = arma::max(vImproved, v);
-
-    iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
-        m / arma::pow(vImproved + epsilon, partial);
-  }
-
   //! Get the value used to initialise the squared gradient parameter.
   double Epsilon() const { return epsilon; }
   //! Modify the value used to initialise the squared gradient parameter.
@@ -121,6 +76,84 @@ class PadamUpdate
   //! Modify the partial adaptive parameter.
   double& Partial() { return partial; }
 
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent Instantiated PadamUpdate parent object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(PadamUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent)
+    {
+      m.zeros(rows, cols);
+      v.zeros(rows, cols);
+      vImproved.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for Padam.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // And update the iterate.
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      v *= parent.beta2;
+      v += (1 - parent.beta2) * (gradient % gradient);
+
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
+          parent.iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
+          parent.iteration);
+
+      // Element wise maximum of past and present squared gradients.
+      vImproved = arma::max(vImproved, v);
+
+      iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
+          m / arma::pow(vImproved + parent.epsilon, parent.partial);
+    }
+
+   private:
+    //! Instantiated parent object.
+    PadamUpdate& parent;
+
+    //! The exponential moving average of gradient values.
+    GradType m;
+
+    //! The exponential moving average of squared gradient values.
+    GradType v;
+
+    //! The optimal sqaured gradient value.
+    GradType vImproved;
+  };
+
  private:
   //! The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
@@ -134,17 +167,8 @@ class PadamUpdate
   //! Partial adaptive parameter.
   double partial;
 
-  //! The exponential moving average of gradient values.
-  arma::mat m;
-
-  //! The exponential moving average of squared gradient values.
-  arma::mat v;
-
-  //! The optimal sqaured gradient value.
-  arma::mat vImproved;
-
   //! The number of iterations.
-  double iteration;
+  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/parallel_sgd/decay_policies/constant_step.hpp

@@ -41,8 +41,9 @@ class ConstantStep
   {
     return step;
   }
+
  private:
-  //! The initial stepsize, which remains unchanged
+  //! The initial stepsize, which remains unchanged.
   double step;
 };
 

include/ensmallen_bits/parallel_sgd/parallel_sgd.hpp

@@ -72,12 +72,37 @@ class ParallelSGD
    * returned.
    *
    * @tparam SparseFunctionType Type of function to be optimized.
+   * @tparam MatType Type of the objective function.
+   * @tparam GradType Type of gradient (it is strongly suggested that this be a
+   *     sparse matrix of some sort!).
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to be optimized(minimized).
    * @param iterate Starting point(will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value at the final point.
    */
-  template <typename SparseFunctionType>
-  double Optimize(SparseFunctionType& function, arma::mat& iterate);
+  template <typename SparseFunctionType,
+            typename MatType,
+            typename GradType,
+            typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SparseFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks);
+
+  //! Forward arma::SpMat<typename MatType::elem_type> as GradType.
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(DecomposableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<DecomposableFunctionType, MatType,
+        arma::SpMat<typename MatType::elem_type>, CallbackTypes...>(
+        function, iterate, std::forward<CallbackTypes>(callbacks)...);
+  }
 
   //! Get the maximum number of iterations (0 indicates no limits).
   size_t MaxIterations() const { return maxIterations; }

include/ensmallen_bits/parallel_sgd/parallel_sgd_impl.hpp

@@ -19,6 +19,32 @@
 
 namespace ens {
 
+// Utility function to update a location of a dense matrix or other type using
+// an atomic section.
+template<typename MatType>
+inline void UpdateLocation(MatType& iterate,
+                           const size_t row,
+                           const size_t col,
+                           const typename MatType::elem_type value)
+{
+  ENS_PRAGMA_OMP_ATOMIC
+  iterate(row, col) -= value;
+}
+
+// Utility function to update a location of a sparse matrix using a critical
+// section.
+template<typename eT>
+inline void UpdateLocation(arma::SpMat<eT>& iterate,
+                           const size_t row,
+                           const size_t col,
+                           const eT value)
+{
+  ENS_PRAGMA_OMP_CRITICAL
+  {
+    iterate(row, col) -= value;
+  }
+}
+
 template <typename DecayPolicyType>
 ParallelSGD<DecayPolicyType>::ParallelSGD(
     const size_t maxIterations,
@@ -34,31 +60,56 @@ ParallelSGD<DecayPolicyType>::ParallelSGD(
 { /* Nothing to do. */ }
 
 template <typename DecayPolicyType>
-template <typename SparseFunctionType>
-double ParallelSGD<DecayPolicyType>::Optimize(
+template <typename SparseFunctionType,
+          typename MatType,
+          typename GradType,
+          typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value,
+typename MatType::elem_type>::type ParallelSGD<DecayPolicyType>::Optimize(
     SparseFunctionType& function,
-    arma::mat& iterate)
+    MatType& iterateIn,
+    CallbackTypes&&... callbacks)
 {
-  // Check that we have all the functions that we need.
-  traits::CheckSparseFunctionTypeAPI<SparseFunctionType>();
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
 
-  double overallObjective = DBL_MAX;
-  double lastObjective;
+  // Check that we have all the functions that we need.
+  traits::CheckSparseFunctionTypeAPI<SparseFunctionType, BaseMatType,
+      BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  ElemType overallObjective = DBL_MAX;
+  ElemType lastObjective;
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
 
   // The order in which the functions will be visited.
+  // TODO: maybe use function.Shuffle() instead?
   arma::Col<size_t> visitationOrder = arma::linspace<arma::Col<size_t>>(0,
       (function.NumFunctions() - 1), function.NumFunctions());
 
   // Iterate till the objective is within tolerance or the maximum number of
   // allowed iterations is reached. If maxIterations is 0, this will iterate
   // till convergence.
-  for (size_t i = 1; i != maxIterations; ++i)
+  terminate |= Callback::BeginOptimization(*this, function, iterate,
+      callbacks...);
+  for (size_t i = 1; i != maxIterations && !terminate; ++i)
   {
     // Calculate the overall objective.
     lastObjective = overallObjective;
 
     overallObjective = function.Evaluate(iterate);
 
+    terminate |= Callback::Evaluate(*this, function, iterate, overallObjective,
+        callbacks...);
+
     // Output current objective function.
     Info << "Parallel SGD: iteration " << i << ", objective "
       << overallObjective << "." << std::endl;
@@ -68,6 +119,8 @@ double ParallelSGD<DecayPolicyType>::Optimize(
       Warn << "Parallel SGD: converged to " << overallObjective
         << "; terminating with failure. Try a smaller step size?"
         << std::endl;
+
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
       return overallObjective;
     }
 
@@ -75,6 +128,8 @@ double ParallelSGD<DecayPolicyType>::Optimize(
     {
       Info << "SGD: minimized within tolerance " << tolerance << "; "
         << "terminating optimization." << std::endl;
+
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
       return overallObjective;
     }
 
@@ -103,29 +158,42 @@ double ParallelSGD<DecayPolicyType>::Optimize(
       {
         // Each instance affects only some components of the decision variable.
         // So the gradient is sparse.
-        arma::sp_mat gradient;
+        BaseGradType gradient;
 
         // Evaluate the sparse gradient.
+        // TODO: support for batch size > 1 could be really useful.
         function.Gradient(iterate, visitationOrder[j], gradient, 1);
 
+        terminate |= Callback::Gradient(*this, function, iterate, gradient,
+            callbacks...);
+
         // Update the decision variable with non-zero components of the
         // gradient.
         for (size_t i = 0; i < gradient.n_cols; ++i)
         {
           // Iterate over the non-zero elements.
-          for (arma::sp_mat::iterator cur = gradient.begin_col(i);
-              cur != gradient.end_col(i); ++cur)
+          const typename GradType::iterator curEnd = gradient.end_col(i);
+          for (typename GradType::iterator cur = gradient.begin_col(i);
+              cur != curEnd; ++cur)
           {
-            ENS_PRAGMA_OMP_ATOMIC
-            iterate(cur.row(), i) -= stepSize * (*cur);
+            const ElemType value = (*cur);
+            const arma::uword row = cur.row();
+
+            // Call out to utility function to use the right type of OpenMP
+            // lock.
+            UpdateLocation(iterate, row, i, stepSize * value);
           }
         }
+        terminate |= Callback::StepTaken(*this, function, iterate,
+            callbacks...);
       }
     }
   }
 
-  Info << "\n Parallel SGD terminated with objective : "
-    << overallObjective << std::endl;
+  Info << "\nParallel SGD terminated with objective : " << overallObjective
+      << "." << std::endl;
+
+  Callback::EndOptimization(*this, function, iterate, callbacks...);
   return overallObjective;
 }
 

include/ensmallen_bits/problems/ackley_function.hpp

@@ -0,0 +1,132 @@
+/**
+ * @file ackley_function.hpp
+ * @author Suryoday Basak
+ *
+ * Definition of the Ackley function.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_PROBLEMS_ACKLEY_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_ACKLEY_FUNCTION_HPP
+
+namespace ens {
+namespace test {
+
+/**
+ * The Ackley function, defined by
+ *
+ * \f[
+ * f(x_1,x_2) = -20 * e^(-0.2 * sqrt(0.5 * (x_1^2 + x_2^2))) -
+ *    e * (0.5(cos(2 * pi * x_1) + cos(2 * pi * x_2))) + e + 20
+ * \f]
+ *
+ * This should optimize to f(x) = 0, at x = [0, 0].
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @book{Ackley1987,
+ *   doi       = {10.1007/978-1-4613-1997-9},
+ *   url       = {https://doi.org/10.1007/978-1-4613-1997-9},
+ *   year      = {1987},
+ *   publisher = {Springer {US}},
+ *   author    = {David H. Ackley},
+ *   title     = {A Connectionist Machine for Genetic Hillclimbing}
+ * }
+ * @endcode
+ */
+class AckleyFunction
+{
+ public:
+  /**
+   * Initialize the AckleyFunction.
+   *
+   * @param c Multiplicative constant with a default value of 2 * pi.
+   * @param epsilon Coefficient to avoid division by zero (numerical stability).
+   */
+  AckleyFunction(const double c = 2 * arma::datum::pi,
+                 const double epsilon = 1e-8);
+
+  /**
+   * Shuffle the order of function visitation. This may be called by the
+   * optimizer.
+   */
+  void Shuffle();
+
+  //! Return 1 (the number of functions).
+  size_t NumFunctions() const { return 1; }
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("-5.0; 5.0"); }
+
+  /**
+   * Evaluate a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param batchSize Number of points to process.
+   */
+  template<typename MatType>
+  typename MatType::elem_type Evaluate(const MatType& coordinates,
+                                       const size_t begin,
+                                       const size_t batchSize) const;
+
+  /**
+   * Evaluate a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   */
+  template<typename MatType>
+  typename MatType::elem_type Evaluate(const MatType& coordinates) const;
+
+  /**
+   * Evaluate the gradient of a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param gradient The function gradient.
+   * @param batchSize Number of points to process.
+   */
+  template<typename MatType, typename GradType>
+  void Gradient(const MatType& coordinates,
+                const size_t begin,
+                GradType& gradient,
+                const size_t batchSize) const;
+
+  /**
+   * Evaluate the gradient of a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   * @param gradient The function gradient.
+   */
+  template<typename MatType, typename GradType>
+  void Gradient(const MatType& coordinates, GradType& gradient);
+
+  //! Get the value used for c.
+  double MultiplicativeConstant() const { return c; }
+  //! Modify the value used for c.
+  double& MultiplicativeConstant() { return c; }
+
+  //! Get the value used for numerical stability.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used for numerical stability.
+  double& Epsilon() { return epsilon; }
+
+ private:
+  //! The value of the multiplicative constant.
+  double c;
+  //! The value used for numerical stability.
+  double epsilon;
+};
+
+} // namespace test
+} // namespace ens
+
+// Include implementation.
+#include "ackley_function_impl.hpp"
+
+#endif // ENSMALLEN_PROBLEMS_ACKLEY_FUNCTION_HPP

include/ensmallen_bits/problems/ackley_function_impl.hpp

@@ -0,0 +1,87 @@
+/**
+ * @file ackley_function_impl.hpp
+ * @author Suryoday Basak
+ *
+ * Implementation of the Ackley function.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_PROBLEMS_ACKLEY_FUNCTION_IMPL_HPP
+#define ENSMALLEN_PROBLEMS_ACKLEY_FUNCTION_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "ackley_function.hpp"
+using namespace std;
+
+namespace ens {
+namespace test {
+
+inline AckleyFunction::AckleyFunction(const double c, const double epsilon) :
+    c(c), epsilon(epsilon)
+{ /* Nothing to do here */}
+
+inline void AckleyFunction::Shuffle() { /* Nothing to do here */ }
+
+template<typename MatType>
+typename MatType::elem_type AckleyFunction::Evaluate(
+    const MatType& coordinates,
+    const size_t /* begin */,
+    const size_t /* batchSize */) const
+{
+  // Convenience typedef.
+  typedef typename MatType::elem_type ElemType;
+
+  // For convenience; we assume these temporaries will be optimized out.
+  const ElemType x1 = coordinates(0);
+  const ElemType x2 = coordinates(1);
+
+  const ElemType objective = -20 * exp(-0.2 * sqrt(0.5 * (x1 * x1 + x2 * x2))) -
+      exp(0.5 * (cos(c * x1) + cos(c * x2))) + exp(1) + 20;
+
+  return objective;
+}
+
+template<typename MatType>
+typename MatType::elem_type AckleyFunction::Evaluate(
+    const MatType& coordinates) const
+{
+  return Evaluate(coordinates, 0, NumFunctions());
+}
+
+template<typename MatType, typename GradType>
+inline void AckleyFunction::Gradient(const MatType& coordinates,
+                                     const size_t /* begin */,
+                                     GradType& gradient,
+                                     const size_t /* batchSize */) const
+{
+  // Convenience typedef.
+  typedef typename MatType::elem_type ElemType;
+
+  // For convenience; we assume these temporaries will be optimized out.
+  const ElemType x1 = coordinates(0);
+  const ElemType x2 = coordinates(1);
+
+  // Aliases for different terms in the expression of the gradient.
+  const ElemType t0 = sqrt(0.5 * (x1 * x1 + x2 * x2));
+  const ElemType t1 = 2.0 * exp(- 0.2 * t0) / (t0 + epsilon);
+  const ElemType t2 = 0.5 * c * exp(0.5 * (cos(c * x1) + cos(c * x2)));
+
+  gradient.set_size(2, 1);
+  gradient(0) = (x1 * t1) + (t2 * sin(c * x1));
+  gradient(1) = (x2 * t1) + (t2 * sin(c * x2));
+}
+
+template<typename MatType, typename GradType>
+inline void AckleyFunction::Gradient(const MatType& coordinates,
+                                     GradType& gradient)
+{
+  Gradient(coordinates, 0, gradient, 1);
+}
+
+} // namespace test
+} // namespace ens
+
+#endif

include/ensmallen_bits/problems/aug_lagrangian_test_functions.hpp

@@ -60,19 +60,31 @@ class GockenbachFunction
 {
  public:
   GockenbachFunction();
-  GockenbachFunction(const arma::mat& initial_point);
+  GockenbachFunction(const arma::mat& initialPoint);
 
-  double Evaluate(const arma::mat& coordinates);
-  void Gradient(const arma::mat& coordinates, arma::mat& gradient);
+  template<typename MatType>
+  typename MatType::elem_type Evaluate(const MatType& coordinates);
+
+  template<typename MatType, typename GradType>
+  void Gradient(const MatType& coordinates, GradType& gradient);
 
   size_t NumConstraints() const { return 2; }
 
-  double EvaluateConstraint(const size_t index, const arma::mat& coordinates);
-  void GradientConstraint(const size_t index,
-                          const arma::mat& coordinates,
-                          arma::mat& gradient);
+  template<typename MatType>
+  typename MatType::elem_type EvaluateConstraint(
+      const size_t index,
+      const MatType& coordinates);
 
-  const arma::mat& GetInitialPoint() const { return initialPoint; }
+  template<typename MatType, typename GradType>
+  void GradientConstraint(const size_t index,
+                          const MatType& coordinates,
+                          GradType& gradient);
+
+  template<typename MatType>
+  MatType GetInitialPoint() const
+  {
+    return arma::conv_to<MatType>::from(initialPoint);
+  }
 
  private:
   arma::mat initialPoint;