Merge pull request #343 from zoq/release-2.19.0

Release version 2.19.0: "Eight Ball Deluxe"

.appveyor.yml

@@ -1,5 +1,5 @@
 environment:
-  ARMADILLO_DOWNLOAD: "http://ftp.fau.de/macports/distfiles/armadillo/armadillo-8.400.0.tar.xz"
+  ARMADILLO_DOWNLOAD: "https://sourceforge.net/projects/arma/files/armadillo-9.800.6.tar.xz"
   BLAS_LIBRARY: "%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a"
   BLAS_LIBRARY_DLL: "%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll"
 
@@ -26,7 +26,7 @@ build_script:
   - cd ..
   - appveyor DownloadFile %ARMADILLO_DOWNLOAD% -FileName armadillo.tar.xz
   - 7z x armadillo.tar.xz -so -txz | 7z x -si -ttar > nul
-  - cd armadillo-8.400.0 && mkdir build && cd build
+  - cd armadillo-9.800.6 && mkdir build && cd build
   - >
     cmake -G "%VSVER%"
     -DBLAS_LIBRARY:FILEPATH=%BLAS_LIBRARY%
@@ -43,7 +43,7 @@ build_script:
   - cd ensmallen && mkdir build && cd build
   - >
     cmake -G "%VSVER%"
-    -DARMADILLO_INCLUDE_DIR=%APPVEYOR_BUILD_FOLDER%/../armadillo-8.400.0/include/
+    -DARMADILLO_INCLUDE_DIR=%APPVEYOR_BUILD_FOLDER%/../armadillo-9.800.6/include/
     -DARMADILLO_LIBRARIES=%BLAS_LIBRARY%
     -DLAPACK_LIBRARY=%BLAS_LIBRARY%
     -DBLAS_LIBRARY=%BLAS_LIBRARY%

.travis.yml

@@ -1,5 +1,5 @@
 os: linux
-dist: trusty
+dist: focal
 language: cpp
 
 env:
@@ -23,10 +23,10 @@ script:
   - if [ $ARMADILLO == "latest" ]; then
       curl https://ftp.fau.de/macports/distfiles/armadillo/`curl https://ftp.fau.de/macports/distfiles/armadillo/ -- | grep '.tar.xz' | sed 's/^.*<a href="\(armadillo-[0-9]*.[0-9]*.[0-9]*.tar.xz\)".*$/\1/' | tail -1` | tar xvJ && cd armadillo*;
     else
-      curl https://ftp.fau.de/macports/distfiles/armadillo/armadillo-8.400.0.tar.xz | tar -xvJ && cd armadillo*;
+      curl -L https://sourceforge.net/projects/arma/files/armadillo-9.800.6.tar.xz | tar -xvJ && cd armadillo*;
     fi
   - cmake . && make && sudo make install && cd ..
-  - mkdir build && cd build && cmake -DCMAKE_CXX_FLAGS="-Werror" -DCMAKE_C_FLAGS="-Werror" .. && make -j2
+  - mkdir build && cd build && cmake .. && make ensmallen_tests -j2
   - CTEST_OUTPUT_ON_FAILURE=1 travis_wait 30 ctest -j2
 
 notifications:

CMakeLists.txt

@@ -2,12 +2,11 @@
 # just installs the headers to the install location, and optionally builds the
 # test program.
 cmake_minimum_required(VERSION 3.3.2)
-project(ensmallen 
+project(ensmallen
         LANGUAGES C CXX)
 
 # Configurable options for CMake.
 option(USE_OPENMP "If available, use OpenMP for parallelization." ON)
-option(BUILD_TESTS "Build tests." ON)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/CMake")
 
@@ -58,7 +57,7 @@ if(USE_OPENMP)
 endif()
 
 # Find Armadillo and link it.
-find_package(Armadillo 8.400.0 REQUIRED)
+find_package(Armadillo 9.800.0 REQUIRED)
 target_link_libraries(ensmallen INTERFACE Armadillo::Armadillo)
 
 # Set helper variables for creating the version, config and target files.
@@ -96,6 +95,4 @@ install(FILES ${CMAKE_SOURCE_DIR}/include/ensmallen.hpp
 
 # Enable testing and build tests.
 enable_testing()
-if (BUILD_TESTS)
-  add_subdirectory(tests)
-endif()
+add_subdirectory(tests)

CONTRIBUTING.md

@@ -108,8 +108,8 @@ $ cd ensmallen
 
 # - or -
 
-$ wget http://ensmallen.org/files/ensmallen-2.16.2.tar.gz
-$ tar -xvzpf ensmallen-2.16.2.tar.gz
+$ wget http://ensmallen.org/files/ensmallen-2.19.0.tar.gz
+$ tar -xvzpf ensmallen-2.19.0.tar.gz
 $ cd ensmallen-latest
 ```
 

HISTORY.md

@@ -1,3 +1,105 @@
+### ensmallen ?.??.?: "???"
+###### ????-??-??
+
+### ensmallen 2.19.0: "Eight Ball Deluxe"
+###### 2022-04-06
+* Added DemonSGD and DemonAdam optimizers
+   ([#211](https://github.com/mlpack/ensmallen/pull/211)).
+
+ * Fix bug with Adam-like optimizers not resetting when `resetPolicy` is `true`.
+   ([#340](https://github.com/mlpack/ensmallen/pull/340)).
+
+ * Add Yogi optimizer
+   ([#232](https://github.com/mlpack/ensmallen/pull/232)).
+
+ * Add AdaBelief optimizer
+   ([#233](https://github.com/mlpack/ensmallen/pull/233)).
+
+ * Add AdaSqrt optimizer
+   ([#234](https://github.com/mlpack/ensmallen/pull/234)).
+   
+ * Bump check for minimum supported version of Armadillo
+   ([#342](https://github.com/mlpack/ensmallen/pull/342)).
+
+### ensmallen 2.18.2: "Fairmount Bagel"
+###### 2022-02-13
+ * Update Catch2 to 2.13.8
+  ([#336](https://github.com/mlpack/ensmallen/pull/336)).
+
+ * Fix epoch timing output
+  ([#337](https://github.com/mlpack/ensmallen/pull/337)).
+
+### ensmallen 2.18.1: "Fairmount Bagel"
+###### 2021-11-19
+ * Accelerate SGD test time
+   ([#330](https://github.com/mlpack/ensmallen/pull/300)).
+
+ * Fix potential infinite loop in CMAES
+   ([#331](https://github.com/mlpack/ensmallen/pull/331)).
+
+ * Fix SCD partial gradient test
+   ([#332](https://github.com/mlpack/ensmallen/pull/332)).
+
+### ensmallen 2.18.0: "Fairmount Bagel"
+###### 2021-10-20
+ * Add gradient value clipping and gradient norm scaling callback
+   ([#315](https://github.com/mlpack/ensmallen/pull/315)).
+
+ * Remove superfluous CMake option to build the tests
+   ([#313](https://github.com/mlpack/ensmallen/pull/313)).
+
+ * Bump minimum Armadillo version to 9.800
+   ([#318](https://github.com/mlpack/ensmallen/pull/318)).
+
+ * Update Catch2 to 2.13.7
+   ([#322](https://github.com/mlpack/ensmallen/pull/322)).
+
+ * Remove redundant template argument for C++20 compatibility
+   ([#324](https://github.com/mlpack/ensmallen/pull/324)).
+
+ * Fix MOEAD test stability
+   ([#327](https://github.com/mlpack/ensmallen/pull/327)).
+
+### ensmallen 2.17.0: "Pachis Din Me Pesa Double"
+###### 2021-07-06
+ * CheckArbitraryFunctionTypeAPI extended for MOO support
+   ([#283](https://github.com/mlpack/ensmallen/pull/283)).
+
+ * Refactor NSGA2
+   ([#263](https://github.com/mlpack/ensmallen/pull/263),
+   [#304](https://github.com/mlpack/ensmallen/pull/304)).
+
+ * Add Indicators for Multiobjective optimizers
+   ([#285](https://github.com/mlpack/ensmallen/pull/285)).
+
+ * Make Callback flexible for MultiObjective Optimizers
+   ([#289](https://github.com/mlpack/ensmallen/pull/289)).
+
+ * Add ZDT Test Suite
+   ([#273](https://github.com/mlpack/ensmallen/pull/273)).
+
+ * Add MOEA-D/DE Optimizer
+   ([#269](https://github.com/mlpack/ensmallen/pull/269)).
+
+ * Introduce Policy Methods for MOEA/D-DE
+   ([#293](https://github.com/mlpack/ensmallen/pull/293)).
+
+ * Add Das-Dennis weight initialization method
+   ([#295](https://github.com/mlpack/ensmallen/pull/295)).
+
+ * Add Dirichlet Weight Initialization
+   ([#296](https://github.com/mlpack/ensmallen/pull/296)).
+
+ * Improved installation and compilation instructions
+   ([#300](https://github.com/mlpack/ensmallen/pull/300)).
+
+ * Disable building the tests by default for faster installation
+   ([#303](https://github.com/mlpack/ensmallen/pull/303)).
+
+ * Modify matrix initialisation to take into account
+   default element zeroing in Armadillo 10.5
+   ([#305](https://github.com/mlpack/ensmallen/pull/305)).
+
 ### ensmallen 2.16.2: "Severely Dented Can Of Polyurethane"
 ###### 2021-03-24
  * Fix CNE test trials

README.md

@@ -2,17 +2,16 @@
   <a href="http://ensmallen.org/"><img src="http://ensmallen.org/img/ensmallen_text.svg" style="background-color:rgba(0,0,0,0);" height=230 alt="ensmallen: a C++ header-only library for numerical optimization"></a>
 </h2>
 
-**ensmallen** is a C++ header-only library for numerical optimization.
+**ensmallen** is a high-quality C++ library for non-linear numerical optimization.
+
+ensmallen provides many types of optimizers that can be used
+for virtually any numerical optimization task.
+This includes gradient descent techniques, gradient-free optimizers,
+and constrained optimization.
+ensmallen also allows optional callbacks to customize the optimization process.
 
 Documentation and downloads: http://ensmallen.org
 
-ensmallen provides a simple set of abstractions for writing an objective
-function to optimize. It also provides a large set of standard and cutting-edge
-optimizers that can be used for virtually any numerical optimization task.
-These include full-batch gradient descent techniques, small-batch techniques,
-gradient-free optimizers, and constrained optimization.
-
-
 ### Requirements
 
 * C++ compiler with C++11 support
@@ -22,13 +21,17 @@ gradient-free optimizers, and constrained optimization.
 
 ### Installation
 
-ensmallen can be installed with CMake 3.3 or later.
-If CMake is not already available on your system, it can be obtained from https://cmake.org
+ensmallen can be installed in several ways: either manually or via cmake, 
+with or without root access.
 
-If you are using an older system such as RHEL 7 or CentOS 7,
-an updated version of CMake is also available via the EPEL repository via the `cmake3` package.
+The cmake based installation will check the requirements 
+and optionally build the tests. If cmake 3.3 (or a later version) 
+is not already available on your system, it can be obtained 
+from [cmake.org](https://cmake.org). If you are using an older 
+system such as RHEL 7 or CentOS 7, an updated version of cmake 
+is also available via the EPEL repository (see the `cmake3` package).
 
-Example installation:
+Example cmake based installation with root access:
 
 ```
 mkdir build
@@ -37,10 +40,54 @@ cmake ..
 sudo make install
 ```
 
+Example cmake based installation without root access, 
+installing into `/home/blah/` (adapt as required):
 
-### Example Usage
+```
+mkdir build
+cd build
+cmake .. -DCMAKE_INSTALL_PREFIX:PATH=/home/blah/
+make install
+```
 
-See [`example.cpp`](example.cpp) for example usage of the L-BFGS optimizer in a linear regression setting.
+The above will create a directory named `/home/blah/include/` 
+and place all ensmallen headers there.
+
+To optionally build and run the tests
+(after running cmake as above),
+use the following additional commands:
+
+```
+make ensmallen_tests
+./ensmallen_tests --durations yes
+```
+
+Manual installation involves simply copying the `include/ensmallen.hpp` header 
+***and*** the associated `include/ensmallen_bits` directory to a location 
+such as `/usr/include/` which is searched by your C++ compiler.
+If you can't use `sudo` or don't have write access to `/usr/include/`, 
+use a directory within your own home directory (eg. `/home/blah/include/`).
+
+
+### Example Compilation
+
+If you have installed ensmallen in a standard location such as `/usr/include/`:
+
+    g++ prog.cpp -o prog -O2 -larmadillo
+    
+If you have installed ensmallen in a non-standard location, 
+such as `/home/blah/include/`, you will need to make sure 
+that your C++ compiler searches `/home/blah/include/` 
+by explicitly specifying the directory as an argument/option. 
+For example, using the `-I` switch in gcc and clang:
+
+    g++ prog.cpp -o prog -O2 -I /home/blah/include/ -larmadillo
+
+
+### Example Optimization
+
+See [`example.cpp`](example.cpp) for example usage of the L-BFGS optimizer 
+in a linear regression setting.
 
 
 ### License
@@ -48,7 +95,7 @@ See [`example.cpp`](example.cpp) for example usage of the L-BFGS optimizer in a
 Unless stated otherwise, the source code for **ensmallen** is licensed under the
 3-clause BSD license (the "License").  A copy of the License is included in the
 "LICENSE.txt" file.  You may also obtain a copy of the License at
-http://opensource.org/licenses/BSD-3-Clause .
+http://opensource.org/licenses/BSD-3-Clause
 
 
 ### Citation
@@ -57,28 +104,20 @@ Please cite the following paper if you use ensmallen in your research and/or
 software. Citations are useful for the continued development and maintenance of
 the library.
 
-* S. Bhardwaj, R. Curtin, M. Edel, Y. Mentekidis, C. Sanderson.  
-  [ensmallen: a flexible C++ library for efficient function optimization](http://www.ensmallen.org/files/ensmallen_2018.pdf).  
-  Workshop on Systems for ML and Open Source Software at NIPS 2018.
+* Ryan R. Curtin, Marcus Edel, Rahul Ganesh Prabhu, Suryoday Basak, Zhihao Lou, Conrad Sanderson.  
+  [The ensmallen library for flexible numerical optimization](https://jmlr.org/papers/volume22/20-416/20-416.pdf).  
+  Journal of Machine Learning Research, Vol. 22, No. 166, 2021.
 
 ```
-@article{DBLP:journals/corr/abs-1810-09361,
-  author    = {Shikhar Bhardwaj and
-               Ryan R. Curtin and
-               Marcus Edel and
-               Yannis Mentekidis and
-               Conrad Sanderson},
-  title     = {ensmallen: a flexible {C++} library for efficient function optimization},
-  journal   = {CoRR},
-  volume    = {abs/1810.09361},
-  doi       = {10.5281/zenodo.2008650},
-  year      = {2018},
-  url       = {http://arxiv.org/abs/1810.09361},
-  archivePrefix = {arXiv},
-  eprint    = {1810.09361},
-  timestamp = {Wed, 31 Oct 2018 14:24:29 +0100},
-  biburl    = {https://dblp.org/rec/bib/journals/corr/abs-1810-09361},
-  bibsource = {dblp computer science bibliography, https://dblp.org}
+@article{ensmallen_JMLR_2021,
+  author  = {Ryan R. Curtin and Marcus Edel and Rahul Ganesh Prabhu and Suryoday Basak and Zhihao Lou and Conrad Sanderson},
+  title   = {The ensmallen library for flexible numerical optimization},
+  journal = {Journal of Machine Learning Research},
+  year    = {2021},
+  volume  = {22},
+  number  = {166},
+  pages   = {1--6},
+  url     = {http://jmlr.org/papers/v22/20-416.html}
 }
 ```
 

doc/callbacks.md

@@ -136,7 +136,7 @@ EarlyStopAtMinLoss cb(
       // You could also, e.g., print the validation loss here to watch it converge.
       return lrfValidation.Evaluate(coordinates);
     });
-    
+
 arma::mat coordinates = lrfTrain.GetInitialPoint();
 SMORMS3 smorms3;
 smorms3.Optimize(lrfTrain, coordinates, cb);
@@ -144,6 +144,89 @@ smorms3.Optimize(lrfTrain, coordinates, cb);
 
 </details>
 
+### GradClipByNorm
+
+One difficulty with optimization is that large parameter gradients can lead an
+optimizer to update the parameters strongly into a region where the loss
+function is much greater, effectively undoing much of the work done to get to
+the current solution. Such large updates during the optimization can cause a
+numerical overflow or underflow, often referred to as "exploding gradients". The
+exploding gradient problem can be caused by: Choosing the wrong learning rate
+which leads to huge updates in the gradients.  Failing to scale a data set
+leading to very large differences between data points.  Applying a loss function
+that computes very large error values.
+
+A common answer to the exploding gradients problem is to change the derivative
+of the error before applying the update step.  One option is to clip the norm
+`||g||` of the gradient `g` before a parameter update. So given the gradient,
+and a maximum norm value, the callback normalizes the gradient so that its
+L2-norm is less than or equal to the given maximum norm value.
+
+#### Constructors
+
+ * `GradClipByNorm(`_`maxNorm`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`maxNorm`** | The maximum clipping value. | |
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+optimizer.Optimize(f, coordinates, GradClipByNorm(0.3));
+```
+
+### GradClipByValue
+
+One difficulty with optimization is that large parameter gradients can lead an
+optimizer to update the parameters strongly into a region where the loss
+function is much greater, effectively undoing much of the work done to get to
+the current solution. Such large updates during the optimization can cause a
+numerical overflow or underflow, often referred to as "exploding gradients". The
+exploding gradient problem can be caused by: Choosing the wrong learning rate
+which leads to huge updates in the gradients.  Failing to scale a data set
+leading to very large differences between data points.  Applying a loss function
+that computes very large error values.
+
+A common answer to the exploding gradients problem is to change the derivative
+of the error before applying the update step.  One option is to clip the
+parameter gradient element-wise before a parameter update.
+
+#### Constructors
+
+ * `GradClipByValue(`_`min, max`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`min`** | The minimum value to clip to. | |
+| `double` | **`max`** | The maximum value to clip to. | |
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+optimizer.Optimize(f, coordinates, GradClipByValue(0, 1.3));
+```
+
 ### PrintLoss
 
 Callback that prints loss to stdout or a specified output stream.
@@ -208,7 +291,7 @@ optimizer.Optimize(f, coordinates, ProgressBar());
 
 </details>
 
-### Report 
+### Report
 
 Callback that prints a optimizer report to stdout or a specified output stream.
 
@@ -472,6 +555,23 @@ an estimate depending on `exactObjective` value.
 | `size_t` | **`epoch`** | The index of the current epoch. |
 | `double` | **`objective`** | Objective value of the current point. |
 
+### GenerationalStepTaken
+
+Called after the evolution of a single generation. Intended specifically for
+MultiObjective Optimizers.
+
+ * `GenerationalStepTaken(`_`optimizer, function, coordinates, objectives, frontIndices`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+| `ObjectivesVecType` | **`objectives`** | The set of calculated objectives so far. |
+| `IndicesType` | **`frontIndices`** | The indices of the members belonging to Pareto Front. |
+
 ## Custom Callbacks
 
 ### Learning rate scheduling

doc/function_types.md

@@ -554,9 +554,11 @@ Each of the implemented methods is allowed to have additional cv-modifiers
 
 The following optimizers can be used with differentiable separable functions:
 
+ - [AdaBelief](#adabelief)
  - [AdaBound](#adabound)
  - [AdaDelta](#adadelta)
  - [AdaGrad](#adagrad)
+ - [AdaSqrt](#adasqrt)
  - [Adam](#adam)
  - [AdaMax](#adamax)
  - [AMSBound](#amsbound)
@@ -876,18 +878,19 @@ NSGA2 nsga;
 double bestFrontSum = nsga.Optimize(objectives, coordinates);
 
 // Set `bestFront` to contain all of the coordinates on the best front.
-std::vector<arma::mat> bestFront = optimizer.Front();
+arma::cube bestFront = optimizer.ParetoFront();
 }
 ```
 
 </details>
 
 *Note*: all multi-objective function optimizers have both the function `Optimize()` to find the
-best front, and also the function `Front()` to return all sets of coordinates that are on the
+best front, and also the function `ParetoFront()` to return all sets of solutions that are on the
 front.
 
 The following optimizers can be used with multi-objective functions:
 - [NSGA2](#nsga2)
+- [MOEA/D-DE](#moead)
 
 ## Constrained functions
 

doc/optimizers.md

@@ -1,3 +1,61 @@
+## AdaBelief
+
+*An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
+
+AdaBelief uses a different denominator from Adam, and is orthogonal to other
+techniques such as recification, decoupled weight decay. The intuition for
+AdaBelief is to adapt the stepsize according to the "belief" in the current
+gradient direction.
+
+#### Constructors
+
+ * `AdaBelief()`
+ * `AdaBelief(`_`stepSize, batchSize`_`)`
+ * `AdaBelief(`_`stepSize, batchSize, beta1, beta2, epsilon, maxIterations, tolerance, shuffle`_`)`
+ * `AdaBelief(`_`stepSize, batchSize, beta1, beta2, epsilon, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`stepSize`** | Step size for each iteration. | `0.001` |
+| `size_t` | **`batchSize`** | Number of points to process in a single step. | `32` |
+| `double` | **`beta1`** | The exponential decay rate for the 1st moment estimates. | `0.9` |
+| `double` | **`beta2`** | The exponential decay rate for the 2nd moment estimates. | `0.999` |
+| `double` | **`epsilon`** | A small constant for numerical stability. | `1e-8` |
+| `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
+| `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
+| `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
+
+The attributes of the optimizer may also be modified via the member methods
+`StepSize()`, `BatchSize()`, `Beta1()`, `Beta2()`, `Epsilon()`, `MaxIterations()`,
+`Tolerance()`, `Shuffle()`, `ResetPolicy()`, and `ExactObjective()`.
+
+#### Examples
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+
+AdaBelief optimizer(0.001, 32, 0.9, 0.999, 1e-12, 100000, 1e-5, true);
+optimizer.Optimize(f, coordinates);
+```
+
+</details>
+
+#### See also:
+
+ * [SGD in Wikipedia](https://en.wikipedia.org/wiki/Stochastic_gradient_descent)
+ * [SGD](#standard-sgd)
+ * [AdaBelief Optimizer: Adapting Stepsizes by the Belief in Observed Gradients](https://arxiv.org/abs/2010.07468)
+ * [Differentiable separable functions](#differentiable-separable-functions)
+
 ## AdaBound
 
 *An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
@@ -168,6 +226,64 @@ optimizer.Optimize(f, coordinates);
 
 </details>
 
+#### See also:
+
+ * [Adaptive Subgradient Methods for Online Learning and Stochastic Optimization](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf)
+ * [AdaGrad in Wikipedia](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#AdaGrad)
+ * [AdaDelta](#adadelta)
+ * [Differentiable separable functions](#differentiable-separable-functions)
+
+## AdaSqrt
+
+*An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
+
+AdaSqrt is an optimizer with parameter-specific learning rates, which are
+adapted relative to how frequently a parameter gets updated during training.
+Larger updates for more sparse parameters and smaller updates for less sparse
+parameters. AdaSqrt, removes the square root in the denominator and scales the
+learning rate by sqrt(T).
+
+#### Constructors
+
+ - `AdaSqrt()`
+ - `AdaSqrt(`_`stepSize`_`)`
+ - `AdaSqrt(`_`stepSize, batchSize`_`)`
+ - `AdaSqrt(`_`stepSize, batchSize, epsilon, maxIterations, tolerance, shuffle`_`)`
+ - `AdaSqrt(`_`stepSize, batchSize, epsilon, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`stepSize`** | Step size for each iteration. | `0.01` |
+| `size_t` | **`batchSize`** | Number of points to process in one step. | `32` |
+| `double` | **`epsilon`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `tolerance` |
+| `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
+| `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
+
+Attributes of the optimizer may also be changed via the member methods
+`StepSize()`, `BatchSize()`, `Epsilon()`, `MaxIterations()`, `Tolerance()`,
+`Shuffle()`, `ResetPolicy()`, and `ExactObjective()`.
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+AdaSqrt optimizer(1.0, 1, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+optimizer.Optimize(f, coordinates);
+```
+
+</details>
+
 #### See also:
 
  * [Adaptive Subgradient Methods for Online Learning and Stochastic Optimization](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf)
@@ -202,14 +318,14 @@ with _`UpdateRule`_` = AdamUpdate`.
 | `double` | **`beta1`** | Exponential decay rate for the first moment estimates. | `0.9` |
 | `double` | **`beta2`** | Exponential decay rate for the weighted infinity norm estimates. | `0.999` |
 | `double` | **`eps`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
-| `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
 | `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
 
 The attributes of the optimizer may also be modified via the member methods
-`StepSize()`, `BatchSize()`, `Beta1()`, `Beta2()`, `Eps()`, `MaxIterations()`,
+`StepSize()`, `BatchSize()`, `Beta1()`, `Beta2()`, `Epsilon()`, `MaxIterations()`,
 `Tolerance()`, `Shuffle()`, `ResetPolicy()`, and `ExactObjective()`.
 
 #### Examples
@@ -260,7 +376,7 @@ with _`UpdateRule`_` = AdaMaxUpdate`.
 | `double` | **`beta1`** | Exponential decay rate for the first moment estimates. | `0.9` |
 | `double` | **`beta2`** | Exponential decay rate for the weighted infinity norm estimates. | `0.999` |
 | `double` | **`eps`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
-| `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
@@ -380,7 +496,7 @@ with _`UpdateRule`_` = AMSGradUpdate`.
 | `double` | **`beta1`** | Exponential decay rate for the first moment estimates. | `0.9` |
 | `double` | **`beta2`** | Exponential decay rate for the weighted infinity norm estimates. | `0.999` |
 | `double` | **`eps`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
-| `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
@@ -744,6 +860,148 @@ optimizer.Optimize(f, coordinates);
  * [Differential Evolution in Wikipedia](https://en.wikipedia.org/wiki/Differential_Evolution)
  * [Arbitrary functions](#arbitrary-functions)
 
+## DemonAdam
+
+*An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
+
+DemonAdam is an Adam based optimizer. DemonAdam is motivated by decaying the
+total contribution of a gradient to all future updates.
+
+#### Constructors
+
+ * `DemonAdam()`
+ * `DemonAdam(`_`stepSize, batchSize`_`)`
+ * `DemonAdam(`_`stepSize, batchSize, momentum, beta1, beta2, eps, maxIterations, tolerance, shuffle`_`)`
+ * `DemonAdam(`_`stepSize, batchSize, momentum, beta1, beta2, eps, maxIterations, tolerance, shuffle, resetPolicy`_`)`
+
+Note that the `DemonAdam` class is based on
+the `DemonAdamType<`_`UpdateRule`_`>` class with _`UpdateRule`_` = AdamUpdate`.
+
+For convenience the following typedefs have been defined:
+
+ * `DemonAdaMax` (equivalent to `DemonAdamType<AdaMaxUpdate>`): DemonAdam that
+   uses the AdaMax update rule.
+ * `DemonAMSGrad` (equivalent to `DemonAdamType<AMSGradUpdate>`): DemonAdam that
+   uses the AMSGrad update rule.
+ * `DemonNadam` (equivalent to `DemonAdamType<NadamUpdate>`): DemonAdam that
+   uses the Nadam update rule.
+ * `NadamUpdate` (equivalent to `DemonAdamType<NadaMaxUpdate>`): DemonAdam that
+   uses the NadaMax update rule.
+ * `DemonOptimisticAdam` (equivalent to `DemonAdamType<OptimisticAdamUpdate>`):
+   DemonAdam that uses the OptimisticAdam update rule.
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`stepSize`** | Step size for each iteration. | `0.001` |
+| `size_t` | **`batchSize`** | Number of points to process in a single step. | `32` |
+| `double` | **`momentum`** | The initial momentum coefficient. | `0.9` |
+| `double` | **`beta1`** | Exponential decay rate for the first moment estimates. | `0.9` |
+| `double` | **`beta2`** | Exponential decay rate for the weighted infinity norm estimates. | `0.999` |
+| `double` | **`eps`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
+| `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
+| `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+
+The attributes of the optimizer may also be modified via the member methods
+`StepSize()`, `BatchSize()`, `Momentum()`, `MomentumIterations()`, `Beta1()`,
+`Beta2()`, `Eps()`, `MaxIterations()`, `Tolerance()`, `Shuffle()`, and
+`ResetPolicy()`.
+
+#### Examples
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+MatyasFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+
+DemonAdam optimizer(0.5, 1, 0.9);
+optimizer.Optimize(f, coordinates);
+```
+
+</details>
+
+#### See also:
+
+ * [SGD in Wikipedia](https://en.wikipedia.org/wiki/Stochastic_gradient_descent)
+ * [SGD](#standard-sgd)
+ * [Decaying momentum helps neural network training](https://arxiv.org/abs/1910.04952)
+ * [Differentiable separable functions](#differentiable-separable-functions)
+
+## DemonSGD
+
+*An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
+
+DemonSGD is an SGD based optimizer. DemonSGD is motivated by decaying the total
+contribution of a gradient to all future updates.
+
+For convenience ensmallen implements various Adam based versions of the Demon
+optimizer:
+
+ * `DemonAdam` (equivalent to `DemonAdamType<AdamUpdate>`): DemonAdam that uses
+   the Adam update rule.
+ * `DemonAdaMax` (equivalent to `DemonAdamType<AdaMaxUpdate>`): DemonAdam that
+   uses the AdaMax update rule.
+ * `DemonAMSGrad` (equivalent to `DemonAdamType<AMSGradUpdate>`): DemonAdam that
+   uses the AMSGrad update rule.
+ * `DemonNadam` (equivalent to `DemonAdamType<NadamUpdate>`): DemonAdam that
+   uses the Nadam update rule.
+ * `NadamUpdate` (equivalent to `DemonAdamType<NadaMaxUpdate>`): DemonAdam that
+   uses the NadaMax update rule.
+ * `DemonOptimisticAdam` (equivalent to `DemonAdamType<OptimisticAdamUpdate>`):
+   DemonAdam that uses the OptimisticAdam update rule.
+
+#### Constructors
+
+ * `DemonSGD()`
+ * `DemonSGD(`_`stepSize, batchSize`_`)`
+ * `DemonSGD(`_`stepSize, batchSize, momentum, maxIterations, tolerance, shuffle`_`)`
+ * `DemonSGD(`_`stepSize, batchSize, momentum, maxIterations, tolerance, shuffle, resetPolicy`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`stepSize`** | Step size for each iteration. | `0.001` |
+| `size_t` | **`batchSize`** | Number of points to process in a single step. | `32` |
+| `double` | **`momentum`** | The initial momentum coefficient. | `0.9` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
+| `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
+| `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+
+The attributes of the optimizer may also be modified via the member methods
+`StepSize()`, `BatchSize()`, `Momentum()`, `MomentumIterations()`,
+`MaxIterations()`, `Tolerance()`, `Shuffle()`, and `ResetPolicy()`.
+
+#### Examples
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+MatyasFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+
+DemonSGD optimizer(0.5, 1, 0.9);
+optimizer.Optimize(f, coordinates);
+```
+
+</details>
+
+#### See also:
+
+ * [SGD in Wikipedia](https://en.wikipedia.org/wiki/Stochastic_gradient_descent)
+ * [SGD](#standard-sgd)
+ * [Decaying momentum helps neural network training](https://arxiv.org/abs/1910.04952)
+ * [Differentiable separable functions](#differentiable-separable-functions)
+
 ## Eve
 
 *An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
@@ -766,8 +1024,8 @@ Eve is a stochastic gradient based optimization method with locally and globally
 | `double` | **`beta2`** | Exponential decay rate for the weighted infinity norm estimates. | `0.999` |
 | `double` | **`beta3`** | Exponential decay rate for relative change. | `0.999` |
 | `double` | **`epsilon`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
-| `double` | **`clip`** | Clipping range to avoid extreme values. | `10` |
-| `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `double` | **`clip`** | Clipping range to avoid extreme valus. | `10` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
@@ -875,7 +1133,7 @@ changes.
 | `double` | **`beta1`** | Exponential decay rate for the first moment estimates. | `0.9` |
 | `double` | **`beta2`** | Exponential decay rate for the weighted infinity norm estimates. | `0.999` |
 | `double` | **`eps`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
-| `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
@@ -1246,7 +1504,7 @@ can be paired with the `Lookahead` optimizer.
 | `BaseOptimizerType` | **`baseOptimizer`** |  Optimizer for the forward step. | Adam |
 | `double` | **`stepSize`** | Step size for each iteration. | `0.5` |
 | `size_t` | **`k`** | The synchronization period. | `5` |
-| `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `DecayPolicyType` | **`decayPolicy`** | Instantiated decay policy used to adjust the step size. | `DecayPolicyType()` |
 | `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
@@ -1408,7 +1666,7 @@ with _`UpdateRule`_` = NadamUpdate`.
 | `double` | **`beta1`** | Exponential decay rate for the first moment estimates. | `0.9` |
 | `double` | **`beta2`** | Exponential decay rate for the weighted infinity norm estimates. | `0.999` |
 | `double` | **`eps`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
-| `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
@@ -1466,7 +1724,7 @@ with _`UpdateRule`_` = NadaMaxUpdate`.
 | `double` | **`beta1`** | Exponential decay rate for the first moment estimates. | `0.9` |
 | `double` | **`beta2`** | Exponential decay rate for the weighted infinity norm estimates. | `0.999` |
 | `double` | **`eps`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
-| `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
@@ -1563,6 +1821,96 @@ optimizer.Optimize(f, coordinates);
  * [SGD in Wikipedia](https://en.wikipedia.org/wiki/Stochastic_gradient_descent)
  * [Differentiable separable functions](#differentiable-separable-functions)
 
+## MOEA/D-DE
+*An optimizer for arbitrary multi-objective functions.*
+MOEA/D-DE (Multi Objective Evolutionary Algorithm based on Decomposition - Differential Evolution) is a multi
+objective optimization algorithm. It works by decomposing the problem into a number of scalar optimization
+subproblems which are solved simultaneously per generation. MOEA/D in itself is a framework, this particular
+algorithm uses Differential Crossover followed by Polynomial Mutation to create offsprings which are then
+decomposed to form a Single Objective Problem. A diversity preserving mechanism is also employed which encourages
+a varied set of solution.
+
+#### Constructors
+* `MOEAD<`_`InitPolicyType, DecompPolicyType`_`>()`
+* `MOEAD<`_`InitPolicyType, DecompPolicyType`_`>(`_`populationSize, maxGenerations, crossoverProb,  neighborProb, neighborSize, distributionIndex, differentialWeight, maxReplace, epsilon, lowerBound, upperBound`_`)`
+
+The _`InitPolicyType`_ template parameter refers to the strategy used to
+initialize the reference directions.
+
+The following types are available:
+
+ * **`Uniform`**
+ * **`BayesianBootstrap`**
+ * **`Dirichlet`**
+
+The _`DecompPolicyType`_ template parameter refers to the strategy used to
+decompose the weight vectors to form a scalar objective function.
+
+The following types are available:
+
+ * **`Tchebycheff`**
+ * **`WeightedAverage`**
+ * **`PenaltyBoundaryIntersection`**
+
+For convenience the following types can be used:
+
+ * **`DefaultMOEAD`** (equivalent to `MOEAD<Uniform, Tchebycheff>`): utilizes Uniform method for weight initialization
+ and Tchebycheff for weight decomposition.
+
+ * **`BBSMOEAD`** (equivalent to `MOEAD<BayesianBootstrap, Tchebycheff>`): utilizes Bayesian Bootstrap method for weight initialization and Tchebycheff for weight decomposition.
+
+ * **`DirichletMOEAD`** (equivalent to `MOEAD<Dirichlet, Tchebycheff>`): utilizes Dirichlet sampling for weight init
+ and Tchebycheff for weight decomposition.
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `size_t` | **`populationSize`** | The number of candidates in the population. | `150` |
+| `size_t` | **`maxGenerations`** | The maximum number of generations allowed. | `300` |
+| `double` | **`crossoverProb`** | Probability that a crossover will occur. | `1.0` |
+| `double` | **`neighborProb`** | The probability of sampling from neighbor. | `0.9` |
+| `size_t` | **`neighborSize`** | The number of nearest-neighbours to consider per weight vector.  | `20` |
+| `double` | **`distributionIndex`** | The crowding degree of the mutation. | `20` |
+| `double` | **`differentialWeight`** | Amplification factor of the differentiation. | `0.5` |
+| `size_t` | **`maxReplace`** | The limit of solutions allowed to be replaced by a child. | `2`|
+| `double` | **`epsilon`** | Handles numerical stability after weight initialization. | `1E-10`|
+| `double`, `arma::vec` | **`lowerBound`** | Lower bound of the coordinates on the coordinates of the whole population during the search process. | `0` |
+| `double`, `arma::vec` | **`upperBound`** | Lower bound of the coordinates on the coordinates of the whole population during the search process. | `1` |
+| `InitPolicyType` | **`initPolicy`** | Instantiated init policy used to initialize weights. | `InitPolicyType()` |
+| `DecompPolicyType` | **`decompPolicy`** | Instantiated decomposition policy used to create scalar objective problem. | `DecompPolicyType()` |
+
+Attributes of the optimizer may also be changed via the member methods
+`PopulationSize()`, `MaxGenerations()`, `CrossoverRate()`, `NeighborProb()`, `NeighborSize()`, `DistributionIndex()`,
+`DifferentialWeight()`, `MaxReplace()`, `Epsilon()`, `LowerBound()`, `UpperBound()`, `InitPolicy()` and `DecompPolicy()`.
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+SchafferFunctionN1<arma::mat> SCH;
+arma::vec lowerBound("-10 -10");
+arma::vec upperBound("10 10");
+DefaultMOEAD opt(300, 300, 1.0, 0.9, 20, 20, 0.5, 2, 1E-10, lowerBound, upperBound);
+typedef decltype(SCH.objectiveA) ObjectiveTypeA;
+typedef decltype(SCH.objectiveB) ObjectiveTypeB;
+arma::mat coords = SCH.GetInitialPoint();
+std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = SCH.GetObjectives();
+// obj will contain the minimum sum of objectiveA and objectiveB found on the best front.
+double obj = opt.Optimize(objectives, coords);
+// Now obtain the best front.
+arma::cube bestFront = opt.ParetoFront();
+```
+</details>
+
+#### See also
+* [MOEA/D-DE Algorithm](https://ieeexplore.ieee.org/document/4633340)
+* [Multi-objective Functions in Wikipedia](https://en.wikipedia.org/wiki/Test_functions_for_optimization#Test_functions_for_multi-objective_optimization)
+* [Multi-objective functions](#multi-objective-functions)
+
 ## NSGA2
 
 *An optimizer for arbitrary multi-objective functions.*
@@ -1619,7 +1967,7 @@ std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = SCH.GetObjectives();
 // obj will contain the minimum sum of objectiveA and objectiveB found on the best front.
 double obj = opt.Optimize(objectives, coords);
 // Now obtain the best front.
-std::vector<arma::mat> bestFront = opt.Front();
+arma::cube bestFront = opt.Front();
 ```
 
 </details>
@@ -1661,7 +2009,7 @@ Note that the `OptimisticAdam` class is based on the
 | `double` | **`beta1`** | Exponential decay rate for the first moment estimates. | `0.9` |
 | `double` | **`beta2`** | Exponential decay rate for the weighted infinity norm estimates. | `0.999` |
 | `double` | **`eps`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
-| `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
@@ -1715,7 +2063,7 @@ Padam is a variant of Adam with a partially adaptive momentum estimation method.
 | `double` | **`beta2`** | Exponential decay rate for the weighted infinity norm estimates. | `0.999` |
 | `double` | **`partial`** | Partially adaptive parameter. | `0.25` |
 | `double` | **`eps`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
-| `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
@@ -2008,7 +2356,7 @@ the following other optimizers:
  | `double` | **`beta1`** | Exponential decay rate for the first moment estimates. | `0.9` |
  | `double` | **`beta2`** | Exponential decay rate for the weighted infinity norm estimates. | `0.999` |
  | `double` | **`eps`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
- | `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+ | `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
  | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
  | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
  | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
@@ -2886,3 +3234,56 @@ optimizer.Optimize(f, coordinates);
  * [SGD](#standard-sgd)
  * [SGD in Wikipedia](https://en.wikipedia.org/wiki/Stochastic_gradient_descent)
  * [Differentiable separable functions](#differentiable-separable-functions)
+
+## Yogi
+
+*An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
+
+Yogi is an optimization algorithm based on Adam with more fine-grained effective
+learning rate control, which uses additive updates instead of multiplicative
+updates for the moving average of the squared gradient. In addition, Yogi has
+similar theoretical guarantees on convergence as Adam.
+
+#### Constructors
+
+ * `Yogi()`
+ * `Yogi(`_`stepSize, batchSize`_`)`
+ * `Yogi(`_`stepSize, batchSize, beta1, beta2, eps, maxIterations`_`)`
+ * `Yogi(`_`stepSize, batchSize, beta1, beta2, eps, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`stepSize`** | Step size for each iteration. | `0.001` |
+| `size_t` | **`batchSize`** | Number of points to process in a single step. | `32` |
+| `double` | **`beta1`** | Exponential decay rate for the first moment estimates. | `0.9` |
+| `double` | **`beta2`** | Exponential decay rate for the weighted infinity norm estimates. | `0.999` |
+| `double` | **`eps`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
+| `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
+| `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
+| `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
+
+The attributes of the optimizer may also be modified via the member methods
+`StepSize()`, `BatchSize()`, `Beta1()`, `Beta2()`, `Eps()`, `MaxIterations()`,
+`Tolerance()`, `Shuffle()`, `ResetPolicy()`, and `ExactObjective()`.
+
+#### Examples
+
+```c++
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+
+Yogi optimizer(0.001, 32, 0.9, 0.999, 1e-8, 100000, 1e-5, true);
+optimizer.Optimize(f, coordinates);
+```
+
+#### See also:
+
+ * [Adaptive Methods for Nonconvex Optimization](https://papers.nips.cc/paper/8186-adaptive-methods-for-nonconvex-optimization)
+ * [SGD in Wikipedia](https://en.wikipedia.org/wiki/Stochastic_gradient_descent)
+ * [SGD](#standard-sgd)
+ * [Adam](#adam)
+ * [Differentiable separable functions](#differentiable-separable-functions)

include/ensmallen.hpp

@@ -29,8 +29,8 @@
   #error "please enable C++11/C++14 mode in your compiler"
 #endif
 
-#if ((ARMA_VERSION_MAJOR < 8) || ((ARMA_VERSION_MAJOR == 8) && (ARMA_VERSION_MINOR < 400)))
-  #error "need Armadillo version 8.400 or later"
+#if ((ARMA_VERSION_MAJOR < 9) || ((ARMA_VERSION_MAJOR == 9) && (ARMA_VERSION_MINOR < 800)))
+  #error "need Armadillo version 9.800 or later"
 #endif
 
 #include <cctype>
@@ -65,6 +65,8 @@
 
 #include "ensmallen_bits/utility/any.hpp"
 #include "ensmallen_bits/utility/arma_traits.hpp"
+#include "ensmallen_bits/utility/indicators/epsilon.hpp"
+#include "ensmallen_bits/utility/indicators/igd_plus.hpp"
 
 // Contains traits, must be placed before report callback.
 #include "ensmallen_bits/function.hpp" // TODO: should move to function/
@@ -72,18 +74,25 @@
 // Callbacks.
 #include "ensmallen_bits/callbacks/callbacks.hpp"
 #include "ensmallen_bits/callbacks/early_stop_at_min_loss.hpp"
+#include "ensmallen_bits/callbacks/grad_clip_by_norm.hpp"
+#include "ensmallen_bits/callbacks/grad_clip_by_value.hpp"
 #include "ensmallen_bits/callbacks/print_loss.hpp"
 #include "ensmallen_bits/callbacks/progress_bar.hpp"
+#include "ensmallen_bits/callbacks/query_front.hpp"
 #include "ensmallen_bits/callbacks/report.hpp"
 #include "ensmallen_bits/callbacks/store_best_coordinates.hpp"
 #include "ensmallen_bits/callbacks/timer_stop.hpp"
 
 #include "ensmallen_bits/problems/problems.hpp" // TODO: should move to another place
 
+#include "ensmallen_bits/ada_belief/ada_belief.hpp"
 #include "ensmallen_bits/ada_bound/ada_bound.hpp"
 #include "ensmallen_bits/ada_delta/ada_delta.hpp"
 #include "ensmallen_bits/ada_grad/ada_grad.hpp"
+#include "ensmallen_bits/ada_sqrt/ada_sqrt.hpp"
 #include "ensmallen_bits/adam/adam.hpp"
+#include "ensmallen_bits/demon_adam/demon_adam.hpp"
+#include "ensmallen_bits/demon_sgd/demon_sgd.hpp"
 #include "ensmallen_bits/qhadam/qhadam.hpp"
 #include "ensmallen_bits/aug_lagrangian/aug_lagrangian.hpp"
 #include "ensmallen_bits/bigbatch_sgd/bigbatch_sgd.hpp"
@@ -100,6 +109,7 @@
 #include "ensmallen_bits/katyusha/katyusha.hpp"
 #include "ensmallen_bits/lbfgs/lbfgs.hpp"
 #include "ensmallen_bits/lookahead/lookahead.hpp"
+#include "ensmallen_bits/moead/moead.hpp"
 #include "ensmallen_bits/nsga2/nsga2.hpp"
 #include "ensmallen_bits/padam/padam.hpp"
 #include "ensmallen_bits/parallel_sgd/parallel_sgd.hpp"
@@ -125,5 +135,6 @@
 #include "ensmallen_bits/svrg/svrg.hpp"
 #include "ensmallen_bits/swats/swats.hpp"
 #include "ensmallen_bits/wn_grad/wn_grad.hpp"
+#include "ensmallen_bits/yogi/yogi.hpp"
 
 #endif

include/ensmallen_bits/ada_belief/ada_belief.hpp

@@ -0,0 +1,186 @@
+/**
+ * @file ada_belief.hpp
+ * @author Marcus Edel
+ *
+ * Class wrapper for the AdaBelief update Policy. The intuition for AdaBelief is
+ * to adapt the stepsize according to the "belief" in the current gradient
+ * direction.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_BELIEF_HPP
+#define ENSMALLEN_ADA_BELIEF_HPP
+
+#include <ensmallen_bits/sgd/sgd.hpp>
+#include "ada_belief_update.hpp"
+
+namespace ens {
+
+/**
+ * The intuition for AdaBelief is to adapt the stepsize according to the
+ * "belief" in the current gradient direction. For more information, see the
+ * following.
+ *
+ * @code
+ * @misc{zhuang2020adabelief,
+ *   title         = {AdaBelief Optimizer: Adapting Stepsizes by the Belief in
+ *                    Observed Gradients},
+ *   author        = {Juntang Zhuang and Tommy Tang and Sekhar Tatikonda and
+ *                    Nicha Dvornek and Yifan Ding and Xenophon Papademetris
+ *                    and James S. Duncan},
+ *   year          = {2020},
+ *   eprint        = {2010.07468},
+ *   archivePrefix = {arXiv},
+ * }
+ * @endcode
+ *
+ * AdaBelief can optimize differentiable separable functions. For more details,
+ * see the documentation on function types included with this distribution or
+ * on the ensmallen website.
+ */
+class AdaBelief
+{
+ public:
+  /**
+   * Construct the AdaBelief optimizer with the given function and parameters.
+   * AdaBelief is sensitive to its parameters and hence a good hyperparameter
+   * selection is necessary as its default may not fit every case.
+   *
+   * The maximum number of iterations refers to the maximum number of
+   * points that are processed (i.e., one iteration equals one point; one
+   * iteration does not equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in a single step.
+   * @param beta1 The exponential decay rate for the 1st moment estimates.
+   * @param beta2 The exponential decay rate for the 2nd moment estimates.
+   * @param epsilon A small constant for numerical stability.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *     limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *     function is visited in linear order.
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *     call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
+   */
+  AdaBelief(const double stepSize = 0.001,
+            const size_t batchSize = 32,
+            const double beta1 = 0.9,
+            const double beta2 = 0.999,
+            const double epsilon = 1e-12,
+            const size_t maxIterations = 100000,
+            const double tolerance = 1e-5,
+            const bool shuffle = true,
+            const bool resetPolicy = true,
+            const bool exactObjective = false);
+
+  /**
+   * Optimize the given function using AdaBelief. The given starting point will
+   * be modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
+  {
+    return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the exponential decay rate for the 1st moment estimates.
+  double Beta1() const { return optimizer.UpdatePolicy().Beta1(); }
+  //! Modify the exponential decay rate for the 1st moment estimates.
+  double& Beta1() { return optimizer.UpdatePolicy().Beta1(); }
+
+  //! Get the exponential decay rate for the 2nd moment estimates.
+  double Beta2() const { return optimizer.UpdatePolicy().Beta2(); }
+  //! Get the second moment coefficient.
+  double& Beta2() { return optimizer.UpdatePolicy().Beta2(); }
+
+  //! Get the value for numerical stability.
+  double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
+  //! Modify the value used for numerical stability.
+  double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
+  //! Get whether or not the update policy parameters are reset before
+  //! Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+  private:
+  //! The Stochastic Gradient Descent object with AdaBelief policy.
+  SGD<AdaBeliefUpdate> optimizer;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "ada_belief_impl.hpp"
+
+#endif

include/ensmallen_bits/ada_belief/ada_belief_impl.hpp

@@ -0,0 +1,44 @@
+/**
+ * @file ada_belief_impl.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of AdaBelief class wrapper.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_BELIEF_ADA_BELIEF_IMPL_HPP
+#define ENSMALLEN_ADA_BELIEF_ADA_BELIEF_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "ada_belief.hpp"
+
+namespace ens {
+
+inline AdaBelief::AdaBelief(
+    const double stepSize,
+    const size_t batchSize,
+    const double beta1,
+    const double beta2,
+    const double epsilon,
+    const size_t maxIterations,
+    const double tolerance,
+    const bool shuffle,
+    const bool resetPolicy,
+    const bool exactObjective) :
+    optimizer(stepSize,
+              batchSize,
+              maxIterations,
+              tolerance,
+              shuffle,
+              AdaBeliefUpdate(epsilon, beta1, beta2),
+              NoDecay(),
+              resetPolicy,
+              exactObjective)
+{ /* Nothing to do. */ }
+
+} // namespace ens
+
+ #endif

include/ensmallen_bits/ada_belief/ada_belief_update.hpp

@@ -0,0 +1,153 @@
+/**
+ * @file ada_belief_update.hpp
+ * @author Marcus Edel
+ *
+ * AdaBelief optimizer update policy. The intuition for AdaBelief is to adapt
+ * the stepsize according to the "belief" in the current gradient direction.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_BELIEF_ADA_BELIEF_UPDATE_HPP
+#define ENSMALLEN_ADA_BELIEF_ADA_BELIEF_UPDATE_HPP
+
+namespace ens {
+
+/**
+ * The intuition for AdaBelief is to adapt the stepsize according to the
+ * "belief" in the current gradient direction.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{zhuang2020adabelief,
+ *   title         = {AdaBelief Optimizer: Adapting Stepsizes by the Belief in
+ *                    Observed Gradients},
+ *   author        = {Juntang Zhuang and Tommy Tang and Sekhar Tatikonda and
+ *                    Nicha Dvornek and Yifan Ding and Xenophon Papademetris
+ *                    and James S. Duncan},
+ *   year          = {2020},
+ *   eprint        = {2010.07468},
+ *   archivePrefix = {arXiv},
+ * }
+ * @endcode
+ */
+class AdaBeliefUpdate
+{
+ public:
+  /**
+   * Construct the AdaBelief update policy with the given parameters.
+   *
+   * @param epsilon A small constant for numerical stability.
+   * @param beta1 The exponential decay rate for the 1st moment estimates.
+   * @param beta2 The exponential decay rate for the 2nd moment estimates.
+   */
+  AdaBeliefUpdate(const double epsilon = 1e-8,
+                  const double beta1 = 0.9,
+                  const double beta2 = 0.999) :
+    epsilon(epsilon),
+    beta1(beta1),
+    beta2(beta2)
+  {
+    // Nothing to do.
+  }
+
+  //! Get the value for numerical stability.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used for numerical stability.
+  double& Epsilon() { return epsilon; }
+
+  //! Get the exponential decay rate for the 1st moment estimates.
+  double Beta1() const { return beta1; }
+  //! Modify the exponential decay rate for the 1st moment estimates.
+  double& Beta1() { return beta1; }
+
+  //! Get the exponential decay rate for the 2nd moment estimates.
+  double Beta2() const { return beta2; }
+  //! Modify the exponential decay rate for the 2nd moment estimates.
+  double& Beta2() { return beta2; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent AdaBeliefUpdate object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AdaBeliefUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent),
+        iteration(0)
+    {
+      m.zeros(rows, cols);
+      s.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for AdaBelief.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Increment the iteration counter variable.
+      ++iteration;
+
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      s *= parent.beta2;
+      s += (1 - parent.beta2) * arma::pow(gradient - m, 2.0) + parent.epsilon;
+
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
+
+      // And update the iterate.
+      iterate -= ((m / biasCorrection1) * stepSize) / (arma::sqrt(s /
+          biasCorrection2) + parent.epsilon);
+    }
+
+   private:
+    //! Instantiated parent object.
+    AdaBeliefUpdate& parent;
+
+    //! The exponential moving average of gradient values.
+    GradType m;
+
+    // The exponential moving average of squared gradient values.
+    GradType s;
+
+    // The number of iterations.
+    size_t iteration;
+  };
+
+ private:
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+
+  // The xponential decay rate for the 1st moment estimates.
+  double beta1;
+
+  // The exponential decay rate for the 2nd moment estimates.
+  double beta2;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/ada_bound/ada_bound_update.hpp

@@ -56,8 +56,7 @@ class AdaBoundUpdate
     gamma(gamma),
     epsilon(epsilon),
     beta1(beta1),
-    beta2(beta2),
-    iteration(0)
+    beta2(beta2)
   {
     // Nothing to do.
   }
@@ -87,11 +86,6 @@ class AdaBoundUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   /**
    * The UpdatePolicyType policy classes must contain an internal 'Policy'
    * template class with two template arguments: MatType and GradType.  This is
@@ -111,7 +105,7 @@ class AdaBoundUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(AdaBoundUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent), first(true), initialStepSize(0)
+        parent(parent), first(true), initialStepSize(0), iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
@@ -139,7 +133,7 @@ class AdaBoundUpdate
       }
 
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // Decay the first and second moment running average coefficient.
       m *= parent.beta1;
@@ -148,16 +142,12 @@ class AdaBoundUpdate
       v *= parent.beta2;
       v += (1 - parent.beta2) * (gradient % gradient);
 
-      const ElemType biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
-      const ElemType biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
+      const ElemType biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
+      const ElemType biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
 
       const ElemType fl = parent.finalLr * stepSize / initialStepSize;
-      const ElemType lower = fl * (1.0 - 1.0 / (parent.gamma *
-          parent.iteration + 1));
-      const ElemType upper = fl * (1.0 + 1.0 / (parent.gamma *
-          parent.iteration));
+      const ElemType lower = fl * (1.0 - 1.0 / (parent.gamma * iteration + 1));
+      const ElemType upper = fl * (1.0 + 1.0 / (parent.gamma * iteration));
 
        // Applies bounds on actual learning rate.
       iterate -= arma::clamp((stepSize *
@@ -180,6 +170,9 @@ class AdaBoundUpdate
 
     // The initial (Adam) learning rate.
     double initialStepSize;
+
+    // The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -197,9 +190,6 @@ class AdaBoundUpdate
 
   // The second moment coefficient.
   double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/ada_bound/ams_bound_update.hpp

@@ -56,8 +56,7 @@ class AMSBoundUpdate
       gamma(gamma),
       epsilon(epsilon),
       beta1(beta1),
-      beta2(beta2),
-      iteration(0)
+      beta2(beta2)
   {
     // Nothing to do.
   }
@@ -87,11 +86,6 @@ class AMSBoundUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   /**
    * The UpdatePolicyType policy classes must contain an internal 'Policy'
    * template class with two template arguments: MatType and GradType.  This is
@@ -111,7 +105,7 @@ class AMSBoundUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(AMSBoundUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent), first(true), initialStepSize(0)
+        parent(parent), first(true), initialStepSize(0), iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
@@ -140,7 +134,7 @@ class AMSBoundUpdate
       }
 
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // Decay the first and second moment running average coefficient.
       m *= parent.beta1;
@@ -149,16 +143,12 @@ class AMSBoundUpdate
       v *= parent.beta2;
       v += (1 - parent.beta2) * (gradient % gradient);
 
-      const ElemType biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
-      const ElemType biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
+      const ElemType biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
+      const ElemType biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
 
       const ElemType fl = parent.finalLr * stepSize / initialStepSize;
-      const ElemType lower = fl * (1.0 - 1.0 / (parent.gamma *
-          parent.iteration + 1));
-      const ElemType upper = fl * (1.0 + 1.0 / (parent.gamma *
-          parent.iteration));
+      const ElemType lower = fl * (1.0 - 1.0 / (parent.gamma * iteration + 1));
+      const ElemType upper = fl * (1.0 + 1.0 / (parent.gamma * iteration));
 
       // Element wise maximum of past and present squared gradients.
       vImproved = arma::max(vImproved, v);
@@ -187,6 +177,9 @@ class AMSBoundUpdate
 
     // The optimal squared gradient value.
     GradType vImproved;
+
+    // The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -204,9 +197,6 @@ class AMSBoundUpdate
 
   // The second moment coefficient.
   double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/ada_sqrt/ada_sqrt.hpp

@@ -0,0 +1,168 @@
+/**
+ * @file ada_sqrt.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of the AdaSqrt optimizer. AdaSqrt is an optimizer that
+ * chooses learning rate dynamically by adapting to the data and iteration.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * terms of the 3-clause BSD license. You should have received a copy of the
+ * 3-clause BSD license along with ensmallen. If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_SQRT_ADA_SQRT_HPP
+#define ENSMALLEN_ADA_SQRT_ADA_SQRT_HPP
+
+#include "../sgd/sgd.hpp"
+#include "ada_sqrt_update.hpp"
+
+namespace ens {
+
+/**
+ * AdaSqrt is a modified version of stochastic gradient descent which performs
+ * larger updates for more sparse parameters and smaller updates for less sparse
+ * parameters.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{hu2019secondorder,
+ *   title  = {Second-order Information in First-order Optimization Methods},
+ *   author = {Yuzheng Hu and Licong Lin and Shange Tang},
+ *   year   = {2019},
+ *   eprint = {1912.09926},
+ * }
+ * @endcode
+ *
+ * AdaSqrt can optimize differentiable separable functions.  For more details,
+ * see the documentation on function types included with this distribution or on
+ * the ensmallen website.
+ */
+class AdaSqrt
+{
+ public:
+  /**
+   * Construct the AdaSqrt optimizer with the given function and parameters.
+   * The defaults here are not necessarily good for the given problem, so it is
+   * suggested that the values used be tailored to the task at hand. The
+   * maximum number of iterations refers to the maximum number of points that
+   * are processed (i.e., one iteration equals one point; one iteration does not
+   * equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in one step.
+   * @param epsilon Value used to initialise the squared gradient parameter.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *        limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *        function is visited in linear order.
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *        call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
+   */
+  AdaSqrt(const double stepSize = 0.01,
+          const size_t batchSize = 32,
+          const double epsilon = 1e-8,
+          const size_t maxIterations = 100000,
+          const double tolerance = 1e-5,
+          const bool shuffle = true,
+          const bool resetPolicy = true,
+          const bool exactObjective = false);
+
+  /**
+   * Optimize the given function using AdaSqrt. The given starting point will
+   * be modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
+  {
+    return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
+  //! Get whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+ private:
+  //! The Stochastic Gradient Descent object with AdaSqrt policy.
+  SGD<AdaSqrtUpdate> optimizer;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "ada_sqrt_impl.hpp"
+
+#endif

include/ensmallen_bits/ada_sqrt/ada_sqrt_impl.hpp

@@ -0,0 +1,38 @@
+/**
+ * @file ada_sqrt_impl.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of AdaSqrt optimizer.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_SQRT_ADA_SQRT_IMPL_HPP
+#define ENSMALLEN_ADA_SQRT_ADA_SQRT_IMPL_HPP
+
+namespace ens {
+
+inline AdaSqrt::AdaSqrt(const double stepSize,
+                        const size_t batchSize,
+                        const double epsilon,
+                        const size_t maxIterations,
+                        const double tolerance,
+                        const bool shuffle,
+                        const bool resetPolicy,
+                        const bool exactObjective) :
+    optimizer(stepSize,
+              batchSize,
+              maxIterations,
+              tolerance,
+              shuffle,
+              AdaSqrtUpdate(epsilon),
+              NoDecay(),
+              resetPolicy,
+              exactObjective)
+{ /* Nothing to do. */ }
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/ada_sqrt/ada_sqrt_update.hpp

@@ -0,0 +1,118 @@
+/**
+ * @file ada_sqrt_update.hpp
+ * @author Marcus Edel
+ *
+ * AdaSqrt update for Stochastic Gradient Descent.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_SQRT_ADA_SQRT_UPDATE_HPP
+#define ENSMALLEN_ADA_SQRT_ADA_SQRT_UPDATE_HPP
+
+namespace ens {
+
+/**
+ * Implementation of the AdaSqrt update policy. AdaSqrt update policy chooses
+ * learning rate dynamically by adapting to the data and iteration.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{hu2019secondorder,
+ *   title  = {Second-order Information in First-order Optimization Methods},
+ *   author = {Yuzheng Hu and Licong Lin and Shange Tang},
+ *   year   = {2019},
+ *   eprint = {1912.09926},
+ * }
+ * @endcode
+ *
+ */
+class AdaSqrtUpdate
+{
+ public:
+  /**
+   * Construct the AdaSqrt update policy with given epsilon parameter.
+   *
+   * @param epsilon The epsilon value used to initialise the squared gradient
+   *        parameter.
+   */
+  AdaSqrtUpdate(const double epsilon = 1e-8) : epsilon(epsilon)
+  {
+    // Nothing to do.
+  }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return epsilon; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD optimizer before the start of the
+     * iteration update process. In AdaSqrt update policy, squared gradient
+     * matrix is initialized to the zeros matrix with the same size as gradient
+     * matrix (see ens::SGD<>).
+     *
+     * @param parent Instantiated parent class.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AdaSqrtUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent),
+        squaredGradient(rows, cols),
+        iteration(0)
+    {
+      // Initialize an empty matrix for sum of squares of parameter gradient.
+      squaredGradient.zeros();
+    }
+
+    /**
+     * Update step for SGD. The AdaSqrt update adapts the learning rate by
+     * performing larger updates for more sparse parameters and smaller updates
+     * for less sparse parameters.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      ++iteration;
+
+      squaredGradient += arma::square(gradient);
+
+      iterate -= stepSize * std::sqrt(iteration) * gradient /
+          (squaredGradient + parent.epsilon);
+    }
+
+   private:
+    // Instantiated parent class.
+    AdaSqrtUpdate& parent;
+    // The squared gradient matrix.
+    GradType squaredGradient;
+    // The number of iterations.
+    size_t iteration;
+  };
+
+ private:
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/adam/adam_update.hpp

@@ -52,8 +52,7 @@ class AdamUpdate
              const double beta2 = 0.999) :
     epsilon(epsilon),
     beta1(beta1),
-    beta2(beta2),
-    iteration(0)
+    beta2(beta2)
   {
     // Nothing to do.
   }
@@ -73,11 +72,6 @@ class AdamUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   /**
    * The UpdatePolicyType policy classes must contain an internal 'Policy'
    * template class with two template arguments: MatType and GradType.  This is
@@ -97,7 +91,8 @@ class AdamUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(AdamUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
@@ -115,7 +110,7 @@ class AdamUpdate
                 const GradType& gradient)
     {
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // And update the iterate.
       m *= parent.beta1;
@@ -124,10 +119,8 @@ class AdamUpdate
       v *= parent.beta2;
       v += (1 - parent.beta2) * (gradient % gradient);
 
-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
 
       /**
        * It should be noted that the term, m / (arma::sqrt(v) + eps), in the
@@ -147,6 +140,9 @@ class AdamUpdate
 
     // The exponential moving average of squared gradient values.
     GradType v;
+
+    // The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -158,9 +154,6 @@ class AdamUpdate
 
   // The second moment coefficient.
   double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/adamax_update.hpp

@@ -54,8 +54,7 @@ class AdaMaxUpdate
                const double beta2 = 0.999) :
     epsilon(epsilon),
     beta1(beta1),
-    beta2(beta2),
-    iteration(0)
+    beta2(beta2)
   {
     // Nothing to do.
   }
@@ -75,11 +74,6 @@ class AdaMaxUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   /**
    * The UpdatePolicyType policy classes must contain an internal 'Policy'
    * template class with two template arguments: MatType and GradType.  This is
@@ -99,7 +93,8 @@ class AdaMaxUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(AdaMaxUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        iteration(0)
     {
       m.zeros(rows, cols);
       u.zeros(rows, cols);
@@ -117,7 +112,7 @@ class AdaMaxUpdate
                 const GradType& gradient)
     {
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // And update the iterate.
       m *= parent.beta1;
@@ -127,8 +122,7 @@ class AdaMaxUpdate
       u *= parent.beta2;
       u = arma::max(u, arma::abs(gradient));
 
-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
 
       if (biasCorrection1 != 0)
         iterate -= (stepSize / biasCorrection1 * m / (u + parent.epsilon));
@@ -141,6 +135,8 @@ class AdaMaxUpdate
     GradType m;
     // The exponentially weighted infinity norm.
     GradType u;
+    // The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -152,9 +148,6 @@ class AdaMaxUpdate
 
   // The second moment coefficient.
   double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/amsgrad_update.hpp

@@ -47,8 +47,7 @@ class AMSGradUpdate
                 const double beta2 = 0.999) :
     epsilon(epsilon),
     beta1(beta1),
-    beta2(beta2),
-    iteration(0)
+    beta2(beta2)
   {
     // Nothing to do.
   }
@@ -68,11 +67,6 @@ class AMSGradUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   /**
    * The UpdatePolicyType policy classes must contain an internal 'Policy'
    * template class with two template arguments: MatType and GradType.  This is
@@ -92,7 +86,8 @@ class AMSGradUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(AMSGradUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
@@ -111,7 +106,7 @@ class AMSGradUpdate
                 const GradType& gradient)
     {
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // And update the iterate.
       m *= parent.beta1;
@@ -120,10 +115,8 @@ class AMSGradUpdate
       v *= parent.beta2;
       v += (1 - parent.beta2) * (gradient % gradient);
 
-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
 
       // Element wise maximum of past and present squared gradients.
       vImproved = arma::max(vImproved, v);
@@ -144,6 +137,9 @@ class AMSGradUpdate
 
     // The optimal squared gradient value.
     GradType vImproved;
+
+    // The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -155,9 +151,6 @@ class AMSGradUpdate
 
   // The second moment coefficient.
   double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/nadam_update.hpp

@@ -50,8 +50,7 @@ class NadamUpdate
       epsilon(epsilon),
       beta1(beta1),
       beta2(beta2),
-      scheduleDecay(scheduleDecay),
-      iteration(0)
+      scheduleDecay(scheduleDecay)
   {
     // Nothing to do.
   }
@@ -76,11 +75,6 @@ class NadamUpdate
   //! Modify the decay parameter for decay coefficients
   double& ScheduleDecay() { return scheduleDecay; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   /**
    * The UpdatePolicyType policy classes must contain an internal 'Policy'
    * template class with two template arguments: MatType and GradType.  This is
@@ -101,7 +95,8 @@ class NadamUpdate
      */
     Policy(NadamUpdate& parent, const size_t rows, const size_t cols) :
         parent(parent),
-        cumBeta1(1)
+        cumBeta1(1),
+        iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
@@ -119,7 +114,7 @@ class NadamUpdate
                 const GradType& gradient)
     {
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // And update the iterate.
       m *= parent.beta1;
@@ -129,18 +124,15 @@ class NadamUpdate
       v += (1 - parent.beta2) * gradient % gradient;
 
       double beta1T = parent.beta1 * (1 - (0.5 *
-          std::pow(0.96, parent.iteration * parent.scheduleDecay)));
+          std::pow(0.96, iteration * parent.scheduleDecay)));
 
       double beta1T1 = parent.beta1 * (1 - (0.5 *
-          std::pow(0.96, (parent.iteration + 1) * parent.scheduleDecay)));
+          std::pow(0.96, (iteration + 1) * parent.scheduleDecay)));
 
       cumBeta1 *= beta1T;
 
       const double biasCorrection1 = 1.0 - cumBeta1;
-
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
-
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
       const double biasCorrection3 = 1.0 - (cumBeta1 * beta1T1);
 
       /* Note :- arma::sqrt(v) + epsilon * sqrt(biasCorrection2) is approximated
@@ -163,6 +155,9 @@ class NadamUpdate
 
     // The cumulative product of decay coefficients.
     double cumBeta1;
+
+    // The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -177,9 +172,6 @@ class NadamUpdate
 
   // The decay parameter for decay coefficients.
   double scheduleDecay;
-
-  // The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/nadamax_update.hpp

@@ -50,8 +50,7 @@ class NadaMaxUpdate
       epsilon(epsilon),
       beta1(beta1),
       beta2(beta2),
-      scheduleDecay(scheduleDecay),
-      iteration(0)
+      scheduleDecay(scheduleDecay)
   {
     // Nothing to do.
   }
@@ -76,11 +75,6 @@ class NadaMaxUpdate
   //! Modify the decay parameter for decay coefficients
   double& ScheduleDecay() { return scheduleDecay; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   /**
    * The UpdatePolicyType policy classes must contain an internal 'Policy'
    * template class with two template arguments: MatType and GradType.  This is
@@ -101,7 +95,8 @@ class NadaMaxUpdate
      */
     Policy(NadaMaxUpdate& parent, const size_t rows, const size_t cols) :
         parent(parent),
-        cumBeta1(1)
+        cumBeta1(1),
+        iteration(0)
     {
       m.zeros(rows, cols);
       u.zeros(rows, cols);
@@ -119,7 +114,7 @@ class NadaMaxUpdate
                 const GradType& gradient)
     {
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // And update the iterate.
       m *= parent.beta1;
@@ -128,10 +123,10 @@ class NadaMaxUpdate
       u = arma::max(u * parent.beta2, arma::abs(gradient));
 
       double beta1T = parent.beta1 * (1 - (0.5 *
-          std::pow(0.96, parent.iteration * parent.scheduleDecay)));
+          std::pow(0.96, iteration * parent.scheduleDecay)));
 
       double beta1T1 = parent.beta1 * (1 - (0.5 *
-          std::pow(0.96, (parent.iteration + 1) * parent.scheduleDecay)));
+          std::pow(0.96, (iteration + 1) * parent.scheduleDecay)));
 
       cumBeta1 *= beta1T;
 
@@ -158,6 +153,9 @@ class NadaMaxUpdate
 
     // The cumulative product of decay coefficients.
     double cumBeta1;
+
+    // The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -172,9 +170,6 @@ class NadaMaxUpdate
 
   // The decay parameter for decay coefficients.
   double scheduleDecay;
-
-  // The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/optimisticadam_update.hpp

@@ -51,8 +51,7 @@ class OptimisticAdamUpdate
                        const double beta2 = 0.999) :
     epsilon(epsilon),
     beta1(beta1),
-    beta2(beta2),
-    iteration(0)
+    beta2(beta2)
   {
     // Nothing to do.
   }
@@ -72,11 +71,6 @@ class OptimisticAdamUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   /**
    * The UpdatePolicyType policy classes must contain an internal 'Policy'
    * template class with two template arguments: MatType and GradType.  This is
@@ -96,7 +90,8 @@ class OptimisticAdamUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(OptimisticAdamUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
@@ -115,7 +110,7 @@ class OptimisticAdamUpdate
                 const GradType& gradient)
     {
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // And update the iterate.
       m *= parent.beta1;
@@ -124,13 +119,10 @@ class OptimisticAdamUpdate
       v *= parent.beta2;
       v += (1 - parent.beta2) * arma::square(gradient);
 
-      GradType mCorrected = m / (1.0 - std::pow(parent.beta1,
-          parent.iteration));
-      GradType vCorrected = v / (1.0 - std::pow(parent.beta2,
-          parent.iteration));
+      GradType mCorrected = m / (1.0 - std::pow(parent.beta1, iteration));
+      GradType vCorrected = v / (1.0 - std::pow(parent.beta2, iteration));
 
-      GradType update = mCorrected /
-          (arma::sqrt(vCorrected) + parent.epsilon);
+      GradType update = mCorrected / (arma::sqrt(vCorrected) + parent.epsilon);
 
       iterate -= (2 * stepSize * update - stepSize * g);
 
@@ -149,6 +141,9 @@ class OptimisticAdamUpdate
 
     // The previous update.
     GradType g;
+
+    // The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -160,9 +155,6 @@ class OptimisticAdamUpdate
 
   // The second moment coefficient.
   double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/callbacks/callbacks.hpp

@@ -744,6 +744,79 @@ class Callback
                     MatType& /* coordinates */)
   { return false; }
 
+ /**
+  * Invoke the GenerationalStepTaken() callback if it exists.
+  * Specialization for MultiObjective case.
+  *
+  * @param callback The callback to call.
+  * @param optimizer The optimizer used to update the function.
+  * @param function Function to optimize.
+  * @param coordinates Starting point.
+  * @param objectives The set of calculated objectives so far.
+  * @param frontIndices The indices of the members belonging to Pareto Front.
+  */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename ObjectivesVecType,
+           typename IndicesType>
+  static typename std::enable_if<
+      callbacks::traits::HasGenerationalStepTakenSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, ObjectivesVecType,
+      IndicesType>::hasBool, bool>::type
+  GenerationalStepTakenFunction(CallbackType& callback,
+                                OptimizerType& optimizer,
+                                FunctionType& function,
+                                MatType& coordinates,
+                                ObjectivesVecType& objectives,
+                                IndicesType& frontIndices)
+  {
+    return const_cast<CallbackType&>(callback).GenerationalStepTaken(
+        optimizer, function, coordinates, objectives, frontIndices);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename ObjectivesVecType,
+           typename IndicesType>
+  static typename std::enable_if<
+      callbacks::traits::HasGenerationalStepTakenSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, ObjectivesVecType,
+      IndicesType>::hasVoid, bool>::type
+  GenerationalStepTakenFunction(CallbackType& callback,
+                                OptimizerType& optimizer,
+                                FunctionType& function,
+                                MatType& coordinates,
+                                ObjectivesVecType& objectives,
+                                IndicesType& frontIndices)
+  {
+    const_cast<CallbackType&>(callback).GenerationalStepTaken(
+        optimizer, function, coordinates, objectives, frontIndices);
+
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename ObjectivesVecType,
+           typename IndicesType>
+  static typename std::enable_if<
+      callbacks::traits::HasGenerationalStepTakenSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, ObjectivesVecType,
+      IndicesType>::hasNone, bool>::type
+  GenerationalStepTakenFunction(CallbackType& /* callback */,
+                                OptimizerType& /* optimizer */,
+                                FunctionType& /* function */,
+                                MatType& /* coordinates */,
+                                ObjectivesVecType& /* objectives */,
+                                IndicesType& /* frontIndices */)
+  { return false; }
+
   /**
    * Iterate over the callbacks and invoke the StepTaken() callback if it
    * exists.
@@ -769,8 +842,41 @@ class Callback
             function, coordinates)... };
      return result;
   }
-};
 
+/**
+ * Iterate over the callbacks and invoke the GenerationalStepTaken() callback if it
+ * exists.
+ *
+ * Specialization for MultiObjective case.
+ *
+ * @param optimizer The optimizer used to update the function.
+ * @param function Function to optimize.
+ * @param coordinates Starting point.
+ * @param objectives The set of calculated objectives so far.
+ * @param frontIndices The indices of the members belonging to Pareto Front.
+ * @param callbacks The callbacks container.
+ */
+template<typename OptimizerType,
+         typename FunctionType,
+         typename ObjectivesVecType,
+         typename IndicesType,
+         typename MatType,
+         typename ...CallbackTypes>
+  static bool GenerationalStepTaken(OptimizerType& optimizer,
+                                    FunctionType& functions,
+                                    MatType& coordinates,
+                                    ObjectivesVecType& objectives,
+                                    IndicesType& frontIndices,
+                                    CallbackTypes&... callbacks)
+{
+  // This will return immediately once a callback returns true.
+  bool result = false;
+  (void)std::initializer_list<bool>{ result =
+    result || Callback::GenerationalStepTakenFunction(callbacks, optimizer,
+      functions, coordinates, objectives, frontIndices)... };
+  return result;
+}
+};
 } // namespace ens
 
 #endif

include/ensmallen_bits/callbacks/early_stop_at_min_loss.hpp

@@ -32,7 +32,7 @@ class EarlyStopAtMinLossType
    * @param patienceIn The number of epochs to wait after the minimum loss has
    *    been reached or no improvement has been made (Default: 10).
    */
-  EarlyStopAtMinLossType<MatType>(const size_t patienceIn = 10) :
+  EarlyStopAtMinLossType(const size_t patienceIn = 10) :
       callbackUsed(false), 
       patience(patienceIn), 
       bestObjective(std::numeric_limits<double>::max()),
@@ -47,7 +47,7 @@ class EarlyStopAtMinLossType
    * @param patienceIn The number of epochs to wait after the minimum loss has
    *    been reached or no improvement has been made (Default: 10).
    */
-  EarlyStopAtMinLossType<MatType>(
+  EarlyStopAtMinLossType(
       std::function<double(const MatType&)> func,
       const size_t patienceIn = 10)
     : callbackUsed(true), 

include/ensmallen_bits/callbacks/grad_clip_by_norm.hpp

@@ -0,0 +1,60 @@
+/**
+ * @file grad_clip_by_norm.hpp
+ * @author Marcus Edel
+ *
+ * Clip the gradients by multiplying the unit vector of the gradients with the
+ * threshold.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_NORM_HPP
+#define ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_NORM_HPP
+
+namespace ens {
+
+/**
+ * Clip the gradients by multiplying the unit vector of the gradients with the
+ * threshold.
+ */
+class GradClipByNorm
+{
+ public:
+  /**
+   * Set up the gradient clip by norm callback class with the maximum clipping
+   * value.
+   *
+   * @param maxNorm The maximum clipping value.
+   */
+  GradClipByNorm(const double maxNorm) : maxNorm(maxNorm)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Callback function called at any call to Gradient().
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param gradient Matrix that holds the gradient.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void Gradient(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                MatType& gradient)
+  {
+    const double gradientNorm = arma::norm(gradient);
+    if (gradientNorm > maxNorm)
+      gradient = maxNorm * gradient / gradientNorm;
+  }
+
+ private:
+  //! The maximum clipping value for gradient clipping.
+  const double maxNorm;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/grad_clip_by_value.hpp

@@ -0,0 +1,60 @@
+/**
+ * @file grad_clip_by_value.hpp
+ * @author Marcus Edel
+ *
+ * Clips the gradient to a specified min and max.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_VALUE_HPP
+#define ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_VALUE_HPP
+
+namespace ens {
+
+/**
+ * Clip the gradient to a specified min and max.
+ */
+class GradClipByValue
+{
+ public:
+  /**
+   * Set up the gradient clip by value callback class with the min and max
+   * value.
+   *
+   * @param min The minimum value to clip to.
+   * @param max The maximum value to clip to.
+   */
+  GradClipByValue(const double min, const double max) : lower(min), upper(max)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Callback function called at any call to Gradient().
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param gradient Matrix that holds the gradient.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void Gradient(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                MatType& gradient)
+  {
+    gradient = arma::clamp(gradient, lower, upper);
+  }
+
+ private:
+  //! The minimum value to clip to.
+  const double lower;
+
+  //! The maximum value to clip to.
+  const double upper;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/progress_bar.hpp

@@ -216,10 +216,10 @@ class ProgressBar
         output << ".";
       }
     }
-
-    const size_t stepTime = epochTimer.toc() / (double) epochSize * 1000;
-    output << "] " << progress << "% - " << (size_t) epochTimer.toc() % 60
-        << "s " << stepTime << "ms/step " << "- loss: " << objective  <<  "\n";
+    const double epochTimerElapsed = epochTimer.toc();
+    const size_t stepTime = epochTimerElapsed / (double) epochSize * 1000;
+    output << "] " << progress << "% - " << epochTimerElapsed
+        << "s/epoch; " << stepTime << "ms/step; loss: " << objective  <<  "\n";
     output.flush();
   }
 

include/ensmallen_bits/callbacks/query_front.hpp

@@ -0,0 +1,85 @@
+/**
+ * @file query_front.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * Implementation of the query front callback function.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_QUERY_FRONT_HPP
+#define ENSMALLEN_CALLBACKS_QUERY_FRONT_HPP
+
+namespace ens {
+
+/**
+ * Query the current Pareto Front after every GenerationalStepTaken callback function.
+ */
+class QueryFront
+{
+ public:
+  /**
+   * Set up the query front callback class with the specified inputs.
+   *
+   * @param queryRate The frequency at which the Pareto Front is queried.
+   * @param paretoFrontArray A reference to a vector of cube to store the queried fronts.
+   */
+    QueryFront(const size_t queryRate, std::vector<arma::cube>& paretoFrontArray) :
+        queryRate(queryRate),
+        paretoFrontArray(paretoFrontArray),
+        genCounter(0)
+    { /* Nothing to do here */ }
+
+   /**
+    * Callback function called at the end of a single generational run.
+    *
+    * @param optimizer The optimizer used to update the function.
+    * @param function Function to optimize.
+    * @param coordinates Starting point.
+    * @param objectives The set of calculated objectives so far.
+    * @param frontIndices The indices of the members belonging to Pareto Front.
+    */
+    template<typename OptimizerType,
+             typename FunctionType,
+             typename MatType,
+             typename ObjectivesVecType,
+             typename IndicesType>
+    void GenerationalStepTaken(OptimizerType& opt,
+                               FunctionType& /* function */,
+                               const MatType& /* coordinates */,
+                               const ObjectivesVecType& objectives,
+                               const IndicesType& frontIndices)
+    {
+      arma::cube currentParetoFront{};
+
+      if (genCounter % queryRate == 0)
+      {
+        currentParetoFront.resize(objectives[0].n_rows, objectives[0].n_cols,
+            frontIndices[0].size());
+        for (size_t solutionIdx = 0; solutionIdx < frontIndices[0].size(); ++solutionIdx)
+        {
+          currentParetoFront.slice(solutionIdx) =
+              arma::conv_to<arma::mat>::from(objectives[frontIndices[0][solutionIdx]]);
+        }
+
+        paretoFrontArray.emplace_back(std::move(currentParetoFront));
+      }
+
+      ++genCounter;
+    }
+
+
+ private:
+  //! The rate of query.
+  size_t queryRate;
+  //! A reference to the array of pareto fronts.
+  std::vector<arma::cube>& paretoFrontArray;
+  //! A counter for the current generation.
+  size_t genCounter;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/traits.hpp

@@ -37,6 +37,8 @@ ENS_HAS_EXACT_METHOD_FORM(BeginEpoch, HasBeginEpoch)
 ENS_HAS_EXACT_METHOD_FORM(EndEpoch, HasEndEpoch)
 //! Detect an StepTaken() method.
 ENS_HAS_EXACT_METHOD_FORM(StepTaken, HasStepTaken)
+//! Detect an GenerationalStepTaken() method.
+ENS_HAS_EXACT_METHOD_FORM(GenerationalStepTaken, HasGenerationalStepTaken)
 
 template<typename OptimizerType,
          typename FunctionType,
@@ -86,6 +88,15 @@ struct TypedForms
                             const MatType&,
                             const MatType&);
 
+  //! This is the form of a bool Gradient() callback method where the gradient
+  //! is modifiable.
+  template<typename CallbackType>
+  using GradientBoolModifiableForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            MatType&);
+
   //! This is the form of a void Gradient() callback method.
   template<typename CallbackType>
   using GradientVoidForm =
@@ -94,6 +105,15 @@ struct TypedForms
                             const MatType&,
                             const MatType&);
 
+  //! This is the form of a void Gradient() callback method where the gradient
+  //! is modifiable.
+  template<typename CallbackType>
+  using GradientVoidModifiableForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            MatType&);
+
   //! This is the form of a bool GradientConstraint() callback method.
   template<typename CallbackType>
   using GradientConstraintBoolForm =
@@ -103,6 +123,16 @@ struct TypedForms
                             const size_t,
                             const MatType&);
 
+  //! This is the form of a bool GradientConstraint() callback method where the
+  //! gradient is modifiable.
+  template<typename CallbackType>
+  using GradientConstraintBoolModifiableForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            MatType&);
+
   //! This is the form of a void GradientConstraint() callback method.
   template<typename CallbackType>
   using GradientConstraintVoidForm =
@@ -112,6 +142,16 @@ struct TypedForms
                             const size_t,
                             const MatType&);
 
+  //! This is the form of a void GradientConstraint() callback method where the
+  //! gradient is modifiable.
+  template<typename CallbackType>
+  using GradientConstraintVoidModifiableForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            MatType&);
+
   //! This is the form of a bool BeginOptimization() callback method.
   template<typename CallbackType>
   using BeginOptimizationBoolForm =
@@ -228,9 +268,9 @@ struct HasEvaluateSignature
 {
   const static bool value =
       HasEvaluate<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EvaluateBoolForm>::value ||
+          FunctionType, MatType>::template EvaluateBoolForm>::value ||
       HasEvaluate<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EvaluateVoidForm>::value;
+          FunctionType, MatType>::template EvaluateVoidForm>::value;
 };
 
 //! Utility struct, check if either void EvaluateConstraint() or
@@ -243,9 +283,9 @@ struct HasEvaluateConstraintSignature
 {
   const static bool value =
       HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EvaluateConstraintBoolForm>::value ||
+          FunctionType, MatType>::template EvaluateConstraintBoolForm>::value ||
       HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EvaluateConstraintVoidForm>::value;
+          FunctionType, MatType>::template EvaluateConstraintVoidForm>::value;
 };
 
 //! Utility struct, check if either void Gradient() or bool Gradient()
@@ -259,9 +299,15 @@ struct HasGradientSignature
 {
   const static bool value =
       HasGradient<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType, Gradient>::template GradientBoolForm>::value ||
+          FunctionType, MatType, Gradient>::template GradientBoolForm>::value ||
       HasGradient<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType, Gradient>::template GradientVoidForm>::value;
+          FunctionType, MatType,
+          Gradient>::template GradientBoolModifiableForm>::value ||
+      HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template GradientVoidForm>::value ||
+      HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType,
+          Gradient>::template GradientVoidModifiableForm>::value;
 };
 
 //! Utility struct, check if either void GradientConstraint() or
@@ -365,6 +411,69 @@ struct HasStepTakenSignature
          FunctionType, MatType>::template StepTakenVoidForm>::value;
 };
 
+//! A utility struct for Typed Forms required in
+//! callbacks for MultiObjective Optimizers.
+template<typename OptimizerType,
+         typename FunctionType,
+         typename MatType,
+         typename ObjectivesVecType,
+         typename IndicesType,
+         typename GradType = MatType>
+struct MOOTypedForms
+{
+  //! This is the form of a bool GenerationalStepTaken() for MOO callback method.
+  template<typename CallbackType>
+  using GenerationalStepTakenBoolForm =
+      bool(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const ObjectivesVecType&,
+                            const IndicesType&);
+
+  //! This is the form of a void StepTaken() for MOO callback method.
+  template<typename CallbackType>
+  using GenerationalStepTakenVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const ObjectivesVecType&,
+                            const IndicesType&);
+};
+
+//! Utility struct, check if either void StepTaken() or bool StepTaken() exists.
+//! Specialization for Multiobjective case.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename ObjectivesVecType,
+         typename IndicesType,
+         typename MatType>
+ struct HasGenerationalStepTakenSignature
+{
+  const static bool hasBool =
+    HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenBoolForm>::value &&
+    !HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenVoidForm>::value;
+
+  const static bool hasVoid =
+    !HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenBoolForm>::value &&
+    HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenVoidForm>::value;
+
+  const static bool hasNone =
+    !HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenBoolForm>::value &&
+    !HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenVoidForm>::value;
+};
 } // namespace traits
 } // namespace callbacks
 } // namespace ens

include/ensmallen_bits/cmaes/cmaes.hpp

@@ -97,9 +97,9 @@ class CMAES
                                        MatType& iterate,
                                        CallbackTypes&&... callbacks);
 
-  //! Get the step size.
+  //! Get the population size.
   size_t PopulationSize() const { return lambda; }
-  //! Modify the step size.
+  //! Modify the population size.
   size_t& PopulationSize() { return lambda; }
 
   //! Get the lower bound of decision variables.

include/ensmallen_bits/cmaes/cmaes_impl.hpp

@@ -159,7 +159,7 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
     // add a small value and try again.
     BaseMatType covLower;
     while (!arma::chol(covLower, C[idx0], "lower"))
-      C[idx0].diag() += 1e-16;
+      C[idx0].diag() += std::numeric_limits<ElemType>::epsilon();
 
     for (size_t j = 0; j < lambda; ++j)
     {

include/ensmallen_bits/demon_adam/demon_adam.hpp

@@ -0,0 +1,207 @@
+/**
+ * @file demon_adam.hpp
+ * @author Marcus Edel
+ *
+ * Definition of DemonAdam.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DEMON_ADAM_DEMON_ADAM_HPP
+#define ENSMALLEN_DEMON_ADAM_DEMON_ADAM_HPP
+
+#include "../sgd/sgd.hpp"
+#include "../adam/adam_update.hpp"
+#include "../adam/adamax_update.hpp"
+#include "../adam/amsgrad_update.hpp"
+#include "../adam/nadam_update.hpp"
+#include "../adam/nadamax_update.hpp"
+#include "../adam/optimisticadam_update.hpp"
+#include "demon_adam_update.hpp"
+
+namespace ens {
+
+/**
+ * DemonAdam automatically decays momentum, motivated by decaying the total
+ * contribution of a gradient to all future updates.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{
+ *   title   = {Decaying momentum helps neural network training},
+ *   author  = {John Chen and Cameron Wolfe and Zhao Li
+ *              and Anastasios Kyrillidis},
+ *   url     = {https://arxiv.org/abs/1910.04952}
+ *   year    = {2019}
+ * }
+ *
+ * DemonAdam can optimize differentiable separable functions. For more details,
+ * see the documentation on function types include with this distribution or on
+ * the ensmallen website.
+ *
+ * @tparam UpdateRule Adam optimizer update rule to be used.
+ */
+template<typename UpdateRule = AdamUpdate>
+class DemonAdamType
+{
+ public:
+  /**
+   * Construct the DemonAdam optimizer with the given function and parameters.
+   * The defaults here are not necessarily good for the given problem, so it is
+   * suggested that the values used be tailored to the task at hand.  The
+   * maximum number of iterations refers to the maximum number of points that
+   * are processed (i.e., one iteration equals one point; one iteration does not
+   * equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in a single step.
+   * @param momentum The initial momentum coefficient.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *     limit).
+   * @param beta1 Exponential decay rate for the first moment estimates.
+   * @param beta2 Exponential decay rate for the weighted infinity norm
+   *     estimates.
+   * @param eps Value used to initialise the mean squared gradient parameter.
+      * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *     function is visited in linear order.
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *     call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *     final objective obtained on the last pass over the data).
+   */
+  DemonAdamType(const double stepSize = 0.001,
+                const size_t batchSize = 32,
+                const double momentum = 0.9,
+                const double beta1 = 0.9,
+                const double beta2 = 0.999,
+                const double eps = 1e-8,
+                const size_t maxIterations = 100000,
+                const double tolerance = 1e-5,
+                const bool shuffle = true,
+                const bool resetPolicy = true,
+                const bool exactObjective = false) :
+      optimizer(stepSize,
+                batchSize,
+                maxIterations,
+                tolerance,
+                shuffle,
+                DemonAdamUpdate<UpdateRule>(maxIterations * batchSize,
+                    momentum, UpdateRule(eps, beta1, beta2)),
+                NoDecay(),
+                resetPolicy,
+                exactObjective)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Optimize the given function using DemonAdam. The given starting point will
+   * be modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return optimizer.template Optimize<
+        SeparableFunctionType, MatType, GradType, CallbackTypes...>(
+        function, iterate, std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the moment coefficient.
+  double Momentum() const { return optimizer.UpdatePolicy().Momentum(); }
+  //! Modify the moment coefficient.
+  double& Momentum() { return optimizer.UpdatePolicy().Momentum(); }
+
+  //! Get the momentum iteration number.
+  size_t MomentumIterations() const
+  { return optimizer.UpdatePolicy().MomentumIterations(); }
+  //! Modify the momentum iteration number.
+  size_t& MomentumIterations()
+  { return optimizer.UpdatePolicy().MomentumIterations(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
+  //! Get whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+ private:
+  //! The Stochastic Gradient Descent object with DemonAdam policy.
+  SGD<DemonAdamUpdate<UpdateRule>> optimizer;
+};
+
+using DemonAdam = DemonAdamType<AdamUpdate>;
+
+using DemonAdaMax = DemonAdamType<AdaMaxUpdate>;
+
+using DemonAMSGrad = DemonAdamType<AMSGradUpdate>;
+
+using DemonNadam = DemonAdamType<NadamUpdate>;
+
+using DemonNadaMax = DemonAdamType<NadaMaxUpdate>;
+
+using DemonOptimisticAdam = DemonAdamType<OptimisticAdamUpdate>;
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/demon_adam/demon_adam_update.hpp

@@ -0,0 +1,169 @@
+/**
+ * @file demon_sgd_update.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of DemonAdam.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DEMON_ADAM_DEMON_ADAM_UPDATE_HPP
+#define ENSMALLEN_DEMON_ADAM_DEMON_ADAM_UPDATE_HPP
+
+#include <assert.h>
+
+namespace ens {
+
+/**
+ * DemonAdam automatically decays momentum, motivated by decaying the total
+ * contribution of a gradient to all future updates.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{
+ *   title   = {Decaying momentum helps neural network training},
+ *   author  = {John Chen and Cameron Wolfe and Zhao Li
+ *              and Anastasios Kyrillidis},
+ *   url     = {https://arxiv.org/abs/1910.04952}
+ *   year    = {2019}
+ * }
+ * @endcode
+ *
+ * @tparam UpdateRule DemonAdam optimizer update rule to be used.
+ */
+template<typename UpdateRule>
+class DemonAdamUpdate
+{
+ public:
+  /**
+   * Construct the DemonAdam update policy with the given parameters.
+   *
+   * @param momentumIterations The number of iterations before the momentum
+   *     will decay to zero.
+   * @param momentum The initial momentum coefficient.
+   * @param adamUpdate Instantiated Adam update policy used to adjust the given
+   *     parameters.
+   */
+  DemonAdamUpdate(const size_t momentumIterations = 100,
+                  const double momentum = 0.9,
+                  const UpdateRule& adamUpdate = UpdateRule()) :
+      T(momentumIterations),
+      betaInit(momentum),
+      t(0),
+      adamUpdateInst(adamUpdate)
+  {
+    // Make sure the momentum iterations parameter is non-zero.
+    assert(momentumIterations != 0 && "The number of iterations before the "
+        "momentum will decay is zero, make sure the max iterations and "
+        "batch size parameter is set correctly. "
+        "Default: momentumIterations = maxIterations / batchSize.");
+  }
+
+  //! Get the momentum coefficient.
+  double Momentum() const { return betaInit; }
+  //! Modify the momentum coefficient.
+  double& Momentum() { return betaInit; }
+
+  //! Get the current iteration number.
+  size_t Iteration() const { return t; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return t; }
+
+  //! Get the momentum ion number.
+  size_t MomentumIterations() const { return T; }
+  //! Modify the momentum iteration number.
+  size_t& MomentumIterations() { return T; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    // Convenient typedef.
+    typedef typename UpdateRule::template Policy<MatType, GradType>
+        InstUpdateRuleType;
+
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent Instantiated PadamUpdate parent object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(DemonAdamUpdate& parent,
+           const size_t rows,
+           const size_t cols) :
+      parent(parent),
+      adamUpdate(new InstUpdateRuleType(parent.adamUpdateInst, rows, cols))
+    { /* Nothing to do here */ }
+
+    /**
+     * Clean any memory associated with the Polciy object.
+     */
+    ~Policy()
+    {
+      delete adamUpdate;
+    }
+
+    /**
+     * Update step for DamonAdam.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      double decayRate = 1;
+      if (parent.t > 0)
+        decayRate = 1.0 - (double) parent.t / (double) parent.T;
+
+      const double betaDecay = parent.betaInit * decayRate;
+      const double beta = betaDecay / ((1.0 - parent.betaInit) + betaDecay);
+
+      // Perform the update.
+      iterate *= beta;
+
+      // Apply the adam update.
+      adamUpdate->Update(iterate, stepSize, gradient);
+
+      // Increment the iteration counter variable.
+      ++parent.t;
+    }
+
+   private:
+    //! Instantiated parent object.
+    DemonAdamUpdate<UpdateRule>& parent;
+
+    //! The update policy.
+    InstUpdateRuleType* adamUpdate;
+  };
+
+ private:
+  //! The number of momentum iterations.
+  size_t T;
+
+  //! Initial momentum coefficient.
+  double betaInit;
+
+  //! The number of iterations.
+  size_t t;
+
+  //! The adam update policy.
+  UpdateRule adamUpdateInst;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/demon_sgd/demon_sgd.hpp

@@ -0,0 +1,178 @@
+/**
+ * @file demon_sgd.hpp
+ * @author Marcus Edel
+ *
+ * Definition of DemonSGD.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DEMON_SGD_DEMON_SGD_HPP
+#define ENSMALLEN_DEMON_SGD_DEMON_SGD_HPP
+
+#include "../sgd/sgd.hpp"
+#include "demon_sgd_update.hpp"
+
+namespace ens {
+
+/**
+ * DemonSGD automatically decays momentum, motivated by decaying the total
+ * contribution of a gradient to all future updates.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{
+ *   title   = {Decaying momentum helps neural network training},
+ *   author  = {John Chen and Cameron Wolfe and Zhao Li
+ *              and Anastasios Kyrillidis},
+ *   url     = {https://arxiv.org/abs/1910.04952}
+ *   year    = {2019}
+ * }
+ *
+ * DemonSGD can optimize differentiable separable functions. For more details,
+ * see the documentation on function types include with this distribution or on
+ * the ensmallen website.
+ */
+class DemonSGD
+{
+ public:
+  /**
+   * Construct the DemonSGD optimizer with the given function and parameters.
+   * The defaults here are not necessarily good for the given problem, so it is
+   * suggested that the values used be tailored to the task at hand.  The
+   * maximum number of iterations refers to the maximum number of points that
+   * are processed (i.e., one iteration equals one point; one iteration does not
+   * equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in a single step.
+   * @param momentum The initial momentum coefficient.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *     limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *     function is visited in linear order.
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *     call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *     final objective obtained on the last pass over the data).
+   */
+  DemonSGD(const double stepSize = 0.001,
+           const size_t batchSize = 32,
+           const double momentum = 0.9,
+           const size_t maxIterations = 100000,
+           const double tolerance = 1e-5,
+           const bool shuffle = true,
+           const bool resetPolicy = true,
+           const bool exactObjective = false) :
+      optimizer(stepSize,
+                batchSize,
+                maxIterations,
+                tolerance,
+                shuffle,
+                DemonSGDUpdate(maxIterations * batchSize, momentum),
+                NoDecay(),
+                resetPolicy,
+                exactObjective)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Optimize the given function using DemonSGD. The given starting point will
+   * be modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return optimizer.template Optimize<
+        SeparableFunctionType, MatType, GradType, CallbackTypes...>(
+        function, iterate, std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the moment coefficient.
+  double Momentum() const { return optimizer.UpdatePolicy().Momentum(); }
+  //! Modify the moment coefficient.
+  double& Momentum() { return optimizer.UpdatePolicy().Momentum(); }
+
+  //! Get the momentum iteration number.
+  size_t MomentumIterations() const
+  { return optimizer.UpdatePolicy().MomentumIterations(); }
+  //! Modify the momentum iteration number.
+  size_t& MomentumIterations()
+  { return optimizer.UpdatePolicy().MomentumIterations(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
+  //! Get whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+ private:
+  //! The Stochastic Gradient Descent object with DemonSGD policy.
+  SGD<DemonSGDUpdate> optimizer;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/demon_sgd/demon_sgd_update.hpp

@@ -0,0 +1,139 @@
+/**
+ * @file demon_sgd_update.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of DemonSGD.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DEMON_SGD_DEMON_SGD_UPDATE_HPP
+#define ENSMALLEN_DEMON_SGD_DEMON_SGD_UPDATE_HPP
+
+namespace ens {
+
+/**
+ * DemonSGD automatically decays momentum, motivated by decaying the total
+ * contribution of a gradient to all future updates.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{
+ *   title   = {Decaying momentum helps neural network training},
+ *   author  = {John Chen and Cameron Wolfe and Zhao Li
+ *              and Anastasios Kyrillidis},
+ *   url     = {https://arxiv.org/abs/1910.04952}
+ *   year    = {2019}
+ * }
+ * @endcode
+ */
+class DemonSGDUpdate
+{
+ public:
+  /**
+   * Construct the DemonSGD update policy with the given parameters.
+   *
+   * @param momentumIterations The number of iterations before the momentum
+   *     will decay to zero.
+   * @param momentum The initial momentum coefficient.
+   */
+  DemonSGDUpdate(const size_t momentumIterations = 100,
+                 const double momentum = 0.9) :
+      T(momentumIterations),
+      betaInit(momentum),
+      t(0)
+  {
+    // Make sure the momentum iterations parameter is non-zero.
+    assert(momentumIterations != 0 && "The number of iterations before the "
+        "momentum will decay is zero, make sure the max iterations and "
+        "batch size parameter is set correctly. "
+        "Default: momentumIterations = maxIterations * batchSize.");
+  }
+
+  //! Get the momentum coefficient.
+  double Momentum() const { return betaInit; }
+  //! Modify the momentum coefficient.
+  double& Momentum() { return betaInit; }
+
+  //! Get the current iteration number.
+  size_t Iteration() const { return t; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return t; }
+
+  //! Get the momentum iteration number.
+  size_t MomentumIterations() const { return T; }
+  //! Modify the momentum iteration number.
+  size_t& MomentumIterations() { return T; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent Instantiated PadamUpdate parent object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(DemonSGDUpdate& parent,
+           const size_t /* rows */,
+           const size_t /* cols */) :
+      parent(parent)
+    { /* Nothing to do here */ }
+
+    /**
+     * Update step for DemonSGD.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      double decayRate = 1;
+      if (parent.t > 0)
+        decayRate = 1.0 - (double) parent.t / (double) parent.T;
+
+      const double betaDecay = parent.betaInit * decayRate;
+      const double beta = betaDecay / ((1.0 - parent.betaInit) + betaDecay);
+
+      // Perform the update.
+      iterate *= beta;
+      iterate -= stepSize * gradient;
+
+      // Increment the iteration counter variable.
+      ++parent.t;
+    }
+
+   private:
+    //! Instantiated parent object.
+    DemonSGDUpdate& parent;
+  };
+
+ private:
+  //! The number of momentum iterations.
+  size_t T;
+
+  //! Initial momentum coefficient.
+  double betaInit;
+
+  //! The number of iterations.
+  size_t t;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/ens_version.hpp

@@ -15,17 +15,17 @@
 #define ENS_VERSION_MAJOR 2
 // The minor version is two digits so regular numerical comparisons of versions
 // work right.  The first minor version of a release is always 10.
-#define ENS_VERSION_MINOR 16
-#define ENS_VERSION_PATCH 2
+#define ENS_VERSION_MINOR 19
+#define ENS_VERSION_PATCH 0
 // If this is a release candidate, it will be reflected in the version name
 // (i.e. the version name will be "RC1", "RC2", etc.).  Otherwise the version
 // name will typically be a seemingly arbitrary set of words that does not
 // contain the capitalized string "RC".
-#define ENS_VERSION_NAME "Severely Dented Can Of Polyurethane"
+#define ENS_VERSION_NAME "Eight Ball Deluxe"
 // Incorporate the date the version was released.
-#define ENS_VERSION_YEAR "2021"
-#define ENS_VERSION_MONTH "03"
-#define ENS_VERSION_DAY "24"
+#define ENS_VERSION_YEAR "2022"
+#define ENS_VERSION_MONTH "04"
+#define ENS_VERSION_DAY "06"
 
 namespace ens {
 

include/ensmallen_bits/ftml/ftml_update.hpp

@@ -50,8 +50,7 @@ class FTMLUpdate
              const double beta2 = 0.999) :
       epsilon(epsilon),
       beta1(beta1),
-      beta2(beta2),
-      iteration(0)
+      beta2(beta2)
   { /* Do nothing. */ }
 
   //! Get the value used to initialise the squared gradient parameter.
@@ -69,11 +68,6 @@ class FTMLUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   /**
    * The UpdatePolicyType policy classes must contain an internal 'Policy'
    * template class with two template arguments: MatType and GradType.  This is
@@ -112,16 +106,14 @@ class FTMLUpdate
                 const GradType& gradient)
     {
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // And update the iterate.
       v *= parent.beta2;
       v += (1 - parent.beta2) * (gradient % gradient);
 
-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
 
       MatType sigma = -parent.beta1 * d;
       d = biasCorrection1 / stepSize *
@@ -145,6 +137,9 @@ class FTMLUpdate
 
     // Parameter update term.
     MatType d;
+
+    // The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -156,9 +151,6 @@ class FTMLUpdate
 
   // The second moment coefficient.
   double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/function/static_checks.hpp

@@ -387,6 +387,25 @@ inline void CheckArbitraryFunctionTypeAPI()
 #endif
 }
 
+template<typename FunctionType, typename... RemainingTypes>
+typename std::enable_if<(sizeof...(RemainingTypes) > 1), void>::type
+CheckArbitraryFunctionTypeAPI()
+{
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  constexpr size_t size = sizeof...(RemainingTypes);
+  using TupleType = typename std::tuple<RemainingTypes...>;
+  using MatType = typename std::tuple_element<size - 1, TupleType>::type;
+
+  static_assert(CheckEvaluate<FunctionType, MatType, MatType>::value,
+      "One of the provided FunctionType does not have a correct definition of Evaluate(). "
+      "Please check that the corresponding FunctionType fully satisfies the requirements "
+      "of the ArbitraryFunctionType API; see the optimizer tutorial for "
+      "more details.");
+
+  CheckArbitraryFunctionTypeAPI<RemainingTypes...>();
+#endif
+}
+
 /**
  * Perform checks for the ResolvableFunctionType API.
  */

include/ensmallen_bits/fw/proximal/proximal_impl.hpp

@@ -45,7 +45,7 @@ inline void Proximal::ProjectToL1Ball(MatType& v, double tau)
   MatType simplexSum = arma::cumsum(simplexSol);
 
   double nu = 0;
-  size_t rho = 0;
+  size_t rho = simplexSol.n_rows - 1;
   for (size_t j = 1; j <= simplexSol.n_rows; j++)
   {
     rho = simplexSol.n_rows - j;
@@ -53,7 +53,7 @@ inline void Proximal::ProjectToL1Ball(MatType& v, double tau)
     if (nu > 0)
       break;
   }
-  double theta = (simplexSum(rho) - tau) / rho;
+  const double theta = (simplexSum(rho) - tau) / rho;
 
   // Threshold on absolute value of v with theta.
   for (arma::uword j = 0; j < simplexSol.n_rows; j++)

include/ensmallen_bits/lbfgs/lbfgs_impl.hpp

@@ -79,7 +79,7 @@ double L_BFGS::ChooseScalingFactor(const size_t iterationNum,
 {
   typedef typename CubeType::elem_type CubeElemType;
 
-  double scalingFactor = 1.0;
+  double scalingFactor;
   if (iterationNum > 0)
   {
     int previousPos = (iterationNum - 1) % numBasis;
@@ -378,7 +378,7 @@ L_BFGS::Optimize(FunctionType& function,
   terminate |= Callback::EvaluateWithGradient(*this, f, iterate,
         functionValue, gradient, callbacks...);
 
-  ElemType prevFunctionValue = functionValue;
+  ElemType prevFunctionValue;
 
   // The main optimization loop.
   terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);

include/ensmallen_bits/moead/decomposition_policies/pbi_decomposition.hpp

@@ -0,0 +1,81 @@
+/**
+ * @file pbi_decomposition.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * The Penalty Based Boundary Intersection (PBI) decomposition policy.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOEAD_PBI_HPP
+#define ENSMALLEN_MOEAD_PBI_HPP
+
+namespace ens {
+
+/**
+ * Penalty Based Boundary Intersection (PBI) method is a weight decomposition method,
+ * it tries to find the intersection between bottom-most boundary of the attainable
+ * objective set with the reference directions.
+ *
+ * The goal is to minimize the distance between objective vectors with the ideal point
+ * along the reference direction. To handle equality constraints, a penalty parameter
+ * theta is used.
+ *
+ * For more information, see the following:
+ * @code
+ * article{zhang2007moea,
+ *   title={MOEA/D: A multiobjective evolutionary algorithm based on decomposition},
+ *   author={Zhang, Qingfu and Li, Hui},
+ *   journal={IEEE Transactions on evolutionary computation},
+ *   pages={712--731},
+ *   year={2007}
+ * @endcode
+ */
+class PenaltyBoundaryIntersection
+{
+ public:
+  /**
+   * Constructor for Penalty Based Boundary Intersection decomposition
+   * policy.
+   *
+   * @param theta The penalty value.
+   */
+  PenaltyBoundaryIntersection(const double theta = 5) :
+      theta(theta)
+  {
+    /* Nothing to do. */
+  }
+
+  /**
+   * Decompose the weight vectors.
+   *
+   * @tparam VecType The type of the vector used in the decommposition.
+   * @param weight The weight vector corresponding to a subproblem.
+   * @param idealPoint The reference point in the objective space.
+   * @param candidateFitness The objective vector of the candidate.
+   */
+  template<typename VecType>
+  typename VecType::elem_type Apply(const VecType& weight,
+                                    const VecType& idealPoint,
+                                    const VecType& candidateFitness)
+  {
+    typedef typename VecType::elem_type ElemType;
+    //! A unit vector in the same direction as the provided weight vector.
+    const VecType referenceDirection = weight / arma::norm(weight);
+    //! Distance of F(x) from the idealPoint along the reference direction.
+    const ElemType d1 = arma::dot(candidateFitness - idealPoint, referenceDirection);
+    //! The perpendicular distance of F(x) from reference direction.
+    const ElemType d2 = arma::norm(candidateFitness - (idealPoint + d1 * referenceDirection));
+
+    return d1 + static_cast<ElemType>(theta) * d2;
+  }
+
+  private:
+    double theta;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/moead/decomposition_policies/tchebycheff_decomposition.hpp

@@ -0,0 +1,66 @@
+/**
+ * @file tchebycheff_decomposition.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * The Tchebycheff Weight decomposition policy.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOEAD_TCHEBYCHEFF_HPP
+#define ENSMALLEN_MOEAD_TCHEBYCHEFF_HPP
+
+namespace ens {
+
+/**
+ * The Tchebycheff method works by taking the maximum of element-wise product
+ * between reference direction and the line connecting objective vector and
+ * ideal point.
+ *
+ * Under mild conditions, for each Pareto Optimal point there exists a reference
+ * direction such that the given point is also the optimal solution
+ * to this scalar objective.
+ *
+ * For more information, see the following:
+ * @code
+ * article{zhang2007moea,
+ *   title={MOEA/D: A multiobjective evolutionary algorithm based on decomposition},
+ *   author={Zhang, Qingfu and Li, Hui},
+ *   journal={IEEE Transactions on evolutionary computation},
+ *   pages={712--731},
+ *   year={2007}
+ * @endcode
+ */
+class Tchebycheff
+{
+ public:
+  /**
+   * Constructor for Tchebycheff decomposition policy.
+   */
+  Tchebycheff()
+  {
+    /* Nothing to do. */
+  }
+
+  /**
+   * Decompose the weight vectors.
+   *
+   * @tparam VecType The type of the vector used in the decommposition.
+   * @param weight The weight vector corresponding to a subproblem.
+   * @param idealPoint The reference point in the objective space.
+   * @param candidateFitness The objective vector of the candidate.
+   */
+  template<typename VecType>
+  typename VecType::elem_type Apply(const VecType& weight,
+                                    const VecType& idealPoint,
+                                    const VecType& candidateFitness)
+  {
+    return arma::max(weight % arma::abs(candidateFitness - idealPoint));
+  }
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/moead/decomposition_policies/weighted_decomposition.hpp

@@ -0,0 +1,62 @@
+/**
+ * @file weighted_decomposition.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * The Weighted Average decomposition policy.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOEAD_WEIGHTED_HPP
+#define ENSMALLEN_MOEAD_WEIGHTED_HPP
+
+namespace ens {
+
+/**
+ * The Weighted average method of decomposition. The working principle is to
+ * minimize the dot product between reference direction and the line connecting
+ * objective vector and ideal point.
+ *
+ * For more information, see the following:
+ * @code
+ * article{zhang2007moea,
+ *   title={MOEA/D: A multiobjective evolutionary algorithm based on decomposition},
+ *   author={Zhang, Qingfu and Li, Hui},
+ *   journal={IEEE Transactions on evolutionary computation},
+ *   pages={712--731},
+ *   year={2007}
+ * @endcode
+ */
+class WeightedAverage
+{
+ public:
+  /**
+   * Constructor for Weighted Average decomposition policy.
+   */
+  WeightedAverage()
+  {
+    /* Nothing to do. */
+  }
+
+  /**
+   * Decompose the weight vectors.
+   *
+   * @tparam VecType The type of the vector used in the decommposition.
+   * @param weight The weight vector corresponding to a subproblem.
+   * @param idealPoint The reference point in the objective space.
+   * @param candidateFitness The objective vector of the candidate.
+   */
+  template<typename VecType>
+  typename VecType::elem_type Apply(const VecType& weight,
+                                    const VecType& /* idealPoint */,
+                                    const VecType& candidateFitness)
+  {
+    return arma::dot(weight, candidateFitness);
+  }
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/moead/moead.hpp

@@ -0,0 +1,338 @@
+/**
+ * @file moead.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * MOEA/D-DE is a multi objective optimization algorithm. MOEA/D-DE
+ * uses genetic algorithms along with a set of reference directions
+ * to drive the population towards the Optimal Front.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_MOEAD_MOEAD_HPP
+#define ENSMALLEN_MOEAD_MOEAD_HPP
+
+//! Decomposition policies.
+#include "decomposition_policies/tchebycheff_decomposition.hpp"
+#include "decomposition_policies/weighted_decomposition.hpp"
+#include "decomposition_policies/pbi_decomposition.hpp"
+
+//! Weight initialization policies.
+#include "weight_init_policies/uniform_init.hpp"
+#include "weight_init_policies/bbs_init.hpp"
+#include "weight_init_policies/dirichlet_init.hpp"
+
+namespace ens {
+
+/**
+ * MOEA/D-DE (Multi Objective Evolutionary Algorithm based on Decompositon - 
+ * Differential Variant) is a multiobjective optimization algorithm. This class 
+ * implements the said optimizer. 
+ *
+ * The algorithm works by generating a candidate population from a fixed starting point. 
+ * Reference directions are generated to guide the optimization process towards the Pareto Front. 
+ * Further, a decomposition function is defined to decompose the problem to a scalar optimization 
+ * objective. Utilizing genetic operators, offsprings are generated with better decomposition values 
+ * to replace the neighboring parent solutions. 
+ *
+ * For more information, see the following:
+ * @code
+ * @article{li2008multiobjective,
+ *   title={Multiobjective optimization problems with complicated Pareto sets, MOEA/D and NSGA-II},
+ *   author={Li, Hui and Zhang, Qingfu},
+ *   journal={IEEE transactions on evolutionary computation},
+ *   pages={284--302},
+ *   year={2008},
+ * @endcode
+ */
+template<typename InitPolicyType = Uniform,
+         typename DecompPolicyType = Tchebycheff>
+class MOEAD {
+ public:
+  /**
+   * Constructor for the MOEA/D optimizer.
+   *
+   * The default values provided here are not necessarily suitable for a
+   * given function. Therefore, it is highly recommended to adjust the
+   * parameters according to the problem.
+   *
+   * @param populationSize The number of elements in the population.
+   * @param maxGenerations The maximum number of generations allowed.
+   * @param crossoverProb The probability that a crossover will occur.
+   * @param neighborProb The probability of sampling from neighbor.
+   * @param neighborSize The number of nearest neighbours of weights
+   *    to find.
+   * @param distributionIndex The crowding degree of the mutation.
+   * @param differentialWeight A parameter used in the mutation of candidate
+   *     solutions controls amplification factor of the differentiation.
+   * @param maxReplace The limit of solutions allowed to be replaced by a child.
+   * @param epsilon Handle numerical stability after weight initialization.
+   * @param lowerBound The lower bound on each variable of a member
+   *    of the variable space.
+   * @param upperBound The upper bound on each variable of a member
+   *    of the variable space.
+   */
+  MOEAD(const size_t populationSize = 300,
+        const size_t maxGenerations = 500,
+        const double crossoverProb = 1.0,
+        const double neighborProb = 0.9,
+        const size_t neighborSize = 20,
+        const double distributionIndex = 20,
+        const double differentialWeight = 0.5,
+        const size_t maxReplace = 2,
+        const double epsilon = 1E-10,
+        const arma::vec& lowerBound = arma::zeros(1, 1),
+        const arma::vec& upperBound = arma::ones(1, 1),
+        const InitPolicyType initPolicy = InitPolicyType(),
+        const DecompPolicyType decompPolicy = DecompPolicyType());
+
+  /**
+   * Constructor for the MOEA/D optimizer. This constructor is provides an
+   * overload to use lowerBound and upperBound as doubles, in case all the
+   * variables in the problem have the same limits.
+   *
+   * The default values provided here are not necessarily suitable for a
+   * given function. Therefore, it is highly recommended to adjust the
+   * parameters according to the problem.
+   *
+   * @param populationSize The number of elements in the population.
+   * @param maxGenerations The maximum number of generations allowed.
+   * @param crossoverProb The probability that a crossover will occur.
+   * @param neighborProb The probability of sampling from neighbor.
+   * @param neighborSize The number of nearest neighbours of weights
+   *    to find.
+   * @param distributionIndex The crowding degree of the mutation.
+   * @param differentialWeight A parameter used in the mutation of candidate
+   *     solutions controls amplification factor of the differentiation.
+   * @param maxReplace The limit of solutions allowed to be replaced by a child.
+   * @param epsilon Handle numerical stability after weight initialization.
+   * @param lowerBound The lower bound on each variable of a member
+   *    of the variable space.
+   * @param upperBound The upper bound on each variable of a member
+   *    of the variable space.
+   */
+    MOEAD(const size_t populationSize = 300,
+          const size_t maxGenerations = 500,
+          const double crossoverProb = 1.0,
+          const double neighborProb = 0.9,
+          const size_t neighborSize = 20,
+          const double distributionIndex = 20,
+          const double differentialWeight = 0.5,
+          const size_t maxReplace = 2,
+          const double epsilon = 1E-10,
+          const double lowerBound = 0,
+          const double upperBound = 1,
+          const InitPolicyType initPolicy = InitPolicyType(),
+          const DecompPolicyType decompPolicy = DecompPolicyType());
+
+  /**
+   * Optimize a set of objectives. The initial population is generated
+   * using the initial point. The output is the best generated front.
+   *
+   * @tparam MatType The type of matrix used to store coordinates.
+   * @tparam ArbitraryFunctionType The type of objective function.
+   * @tparam CallbackTypes Types of callback function.
+   * @param objectives std::tuple of the objective functions.
+   * @param iterate The initial reference point for generating population.
+   * @param callbacks The callback functions.
+   */
+  template<typename MatType,
+           typename... ArbitraryFunctionType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(std::tuple<ArbitraryFunctionType...>& objectives,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks);
+
+  //! Retrieve population size.
+  size_t PopulationSize() const { return populationSize; }
+  //! Modify the population size.
+  size_t& PopulationSize() { return populationSize; }
+
+  //! Retrieve number of generations.
+  size_t MaxGenerations() const { return maxGenerations; }
+  //! Modify the number of generations.
+  size_t& MaxGenerations() { return maxGenerations; }
+
+  //! Retrieve crossover rate.
+  double CrossoverRate() const { return crossoverProb; }
+  //! Modify the crossover rate.
+  double& CrossoverRate() { return crossoverProb; }
+
+  //! Retrieve size of the weight neighbor.
+  size_t NeighborSize() const { return neighborSize; }
+  //! Modify the size of the weight neighbor.
+  size_t& NeighborSize() { return neighborSize; }
+
+  //! Retrieve value of the distribution index.
+  double DistributionIndex() const { return distributionIndex; }
+  //! Modify the value of the distribution index.
+  double& DistributionIndex() { return distributionIndex; }
+
+  //! Retrieve value of neighbor probability.
+  double NeighborProb() const { return neighborProb; }
+  //! Modify the value of neigbourhood probability.
+  double& NeighborProb() { return neighborProb; }
+
+  //! Retrieve value of scaling factor.
+  double DifferentialWeight() const { return differentialWeight; }
+  //! Modify the value of scaling factor.
+  double& DifferentialWeight() { return differentialWeight; }
+
+  //! Retrieve value of maxReplace.
+  size_t MaxReplace() const { return maxReplace; }
+  //! Modify value of maxReplace.
+  size_t& MaxReplace() { return maxReplace; }
+
+   //! Retrieve value of epsilon.
+  double Epsilon() const { return epsilon; }
+  //! Modify value of maxReplace.
+  double& Epsilon() { return epsilon; }
+
+  //! Retrieve value of lowerBound.
+  const arma::vec& LowerBound() const { return lowerBound; }
+  //! Modify value of lowerBound.
+  arma::vec& LowerBound() { return lowerBound; }
+
+  //! Retrieve value of upperBound.
+  const arma::vec& UpperBound() const { return upperBound; }
+  //! Modify value of upperBound.
+  arma::vec& UpperBound() { return upperBound; }
+
+  //! Retrieve the Pareto optimal points in variable space. This returns an empty cube
+  //! until `Optimize()` has been called.
+  const arma::cube& ParetoSet() const { return paretoSet; }
+
+  //! Retrieve the best front (the Pareto frontier). This returns an empty cube until
+  //! `Optimize()` has been called.
+  const arma::cube& ParetoFront() const { return paretoFront; }
+
+  //! Get the weight initialization policy.
+  const InitPolicyType& InitPolicy() const { return initPolicy; }
+  //! Modify the weight initialization policy.
+  InitPolicyType& InitPolicy() { return initPolicy; }
+
+  //! Get the weight decomposition policy.
+  const DecompPolicyType& DecompPolicy() const { return decompPolicy; }
+  //! Modify the weight decomposition policy.
+  DecompPolicyType& DecompPolicy() { return decompPolicy; }
+
+ private:
+  /**
+   * @brief Randomly selects two members from the population.
+   *
+   * @param subProblemIdx Index of the current subproblem.
+   * @param neighborSize A matrix containing indices of the neighbors.
+   * @return std::tuple<size_t, size_t> The chosen pair of indices.
+   */
+  std::tuple<size_t, size_t> Mating(size_t subProblemIdx,
+                                    const arma::umat& neighborSize,
+                                    bool sampleNeighbor);
+
+  /**
+   * Mutate the child formed by the crossover of two random members of the
+   * population. Uses polynomial mutation.
+   *
+   * @tparam MatType The type of matrix used to store coordinates.
+   * @param child The candidate to be mutated.
+   * @param mutationRate The probability of mutation.
+   * @param lowerBound The lower bound on each variable in the matrix.
+   * @param upperBound The upper bound on each variable in the matrix.
+   * @return The mutated child.
+   */
+  template<typename MatType>
+  void Mutate(MatType& child,
+              double mutationRate,
+              const MatType& lowerBound,
+              const MatType& upperBound);
+
+  /**
+   * Evaluate objectives for the elite population.
+   *
+   * @tparam ArbitraryFunctionType std::tuple of multiple function types.
+   * @tparam MatType Type of matrix to optimize.
+   * @param population The elite population.
+   * @param objectives The set of objectives.
+   * @param calculatedObjectives Vector to store calculated objectives.
+   */
+  template<std::size_t I = 0,
+           typename MatType,
+           typename ...ArbitraryFunctionType>
+  typename std::enable_if<I == sizeof...(ArbitraryFunctionType), void>::type
+  EvaluateObjectives(
+                     std::vector<MatType>&,
+                     std::tuple<ArbitraryFunctionType...>&,
+                     std::vector<arma::Col<typename MatType::elem_type> >&);
+
+  template<std::size_t I = 0,
+           typename MatType,
+           typename ...ArbitraryFunctionType>
+  typename std::enable_if<I < sizeof...(ArbitraryFunctionType), void>::type
+  EvaluateObjectives(
+                     std::vector<MatType>& population,
+                     std::tuple<ArbitraryFunctionType...>& objectives,
+                     std::vector<arma::Col<typename MatType::elem_type> >&
+                     calculatedObjectives);
+
+  //! Size of the population.
+  size_t populationSize;
+
+  //! Maximum number of generations before termination criteria is met.
+  size_t maxGenerations;
+
+  //! Probability of crossover between two members.
+  double crossoverProb;
+
+  //! The probability that two elements will be chosen from the neighbor.
+  double neighborProb;
+
+  //! Number of nearest neighbours of weights to consider.
+  size_t neighborSize;
+
+  //! The crowding degree of the mutation. Higher value produces a mutant
+  //! resembling its parent.
+  double distributionIndex;
+
+  //! Amplification factor for differentiation.
+  double differentialWeight;
+
+  //! Maximum number of childs which can replace the parent. Higher value
+  //! leads to a loss of diversity.
+  size_t maxReplace;
+
+  //! A small numeric value to be added to the weights after initialization.
+  //! Prevents zero value inside inited weights.
+  double epsilon;
+
+  //! Lower bound on each variable in the variable space.
+  arma::vec lowerBound;
+
+  //! Upper bound on each variable in the variable space.
+  arma::vec upperBound;
+
+  //! The set of all the Pareto optimal points.
+  //! Stored after Optimize() is called.
+  arma::cube paretoSet;
+
+  //! The set of all the Pareto optimal objective vectors.
+  //! Stored after Optimize() is called.
+  arma::cube paretoFront;
+
+  //! Policy to initialize the reference directions (weights) matrix.
+  InitPolicyType initPolicy;
+
+  //! Policy to decompose the weights.
+  DecompPolicyType decompPolicy;
+};
+
+using DefaultMOEAD = MOEAD<Uniform, Tchebycheff>;
+using BBSMOEAD = MOEAD<BayesianBootstrap, Tchebycheff>;
+using DirichletMOEAD = MOEAD<Dirichlet, Tchebycheff>;
+} // namespace ens
+
+// Include implementation.
+#include "moead_impl.hpp"
+
+#endif

include/ensmallen_bits/moead/moead_impl.hpp

@@ -0,0 +1,441 @@
+/**
+ * @file moead_impl.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * Implementation of the MOEA/D-DE algorithm. Used for multi-objective
+ * optimization problems on arbitrary functions.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more Information.
+ */
+
+#ifndef ENSMALLEN_MOEAD_MOEAD_IMPL_HPP
+#define ENSMALLEN_MOEAD_MOEAD_IMPL_HPP
+
+#include "moead.hpp"
+#include <assert.h>
+
+namespace ens {
+template <typename InitPolicyType, typename DecompPolicyType>
+inline MOEAD<InitPolicyType, DecompPolicyType>::
+MOEAD(const size_t populationSize,
+      const size_t maxGenerations,
+      const double crossoverProb,
+      const double neighborProb,
+      const size_t neighborSize,
+      const double distributionIndex,
+      const double differentialWeight,
+      const size_t maxReplace,
+      const double epsilon,
+      const arma::vec& lowerBound,
+      const arma::vec& upperBound,
+      const InitPolicyType initPolicy,
+      const DecompPolicyType decompPolicy) :
+    populationSize(populationSize),
+    maxGenerations(maxGenerations),
+    crossoverProb(crossoverProb),
+    neighborProb(neighborProb),
+    neighborSize(neighborSize),
+    distributionIndex(distributionIndex),
+    differentialWeight(differentialWeight),
+    maxReplace(maxReplace),
+    epsilon(epsilon),
+    lowerBound(lowerBound),
+    upperBound(upperBound),
+    initPolicy(initPolicy),
+    decompPolicy(decompPolicy)
+  { /* Nothing to do here. */ }
+
+template <typename InitPolicyType, typename DecompPolicyType>
+inline MOEAD<InitPolicyType, DecompPolicyType>::
+MOEAD(const size_t populationSize,
+      const size_t maxGenerations,
+      const double crossoverProb,
+      const double neighborProb,
+      const size_t neighborSize,
+      const double distributionIndex,
+      const double differentialWeight,
+      const size_t maxReplace,
+      const double epsilon,
+      const double lowerBound,
+      const double upperBound,
+      const InitPolicyType initPolicy,
+      const DecompPolicyType decompPolicy) :
+    populationSize(populationSize),
+    maxGenerations(maxGenerations),
+    crossoverProb(crossoverProb),
+    neighborProb(neighborProb),
+    neighborSize(neighborSize),
+    distributionIndex(distributionIndex),
+    differentialWeight(differentialWeight),
+    maxReplace(maxReplace),
+    epsilon(epsilon),
+    lowerBound(lowerBound * arma::ones(1, 1)),
+    upperBound(upperBound * arma::ones(1, 1)),
+    initPolicy(initPolicy),
+    decompPolicy(decompPolicy)
+  { /* Nothing to do here. */ }
+
+//! Optimize the function.
+template <typename InitPolicyType, typename DecompPolicyType>
+template<typename MatType,
+         typename... ArbitraryFunctionType,
+         typename... CallbackTypes>
+typename MatType::elem_type MOEAD<InitPolicyType, DecompPolicyType>::
+Optimize(std::tuple<ArbitraryFunctionType...>& objectives,
+         MatType& iterateIn,
+         CallbackTypes&&... callbacks)
+{
+  // Population Size must be at least 3 for MOEA/D-DE to work.
+  if (populationSize < 3)
+  {
+    throw std::logic_error("MOEA/D-DE::Optimize(): population size should be at least"
+        " 3!");
+  }
+
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  // Make sure that we have the methods that we need.  Long name...
+  traits::CheckArbitraryFunctionTypeAPI<ArbitraryFunctionType...,
+      BaseMatType>();
+  RequireDenseFloatingPointType<BaseMatType>();
+
+  if (neighborSize < 2)
+  {
+    throw std::invalid_argument(
+        "neighborSize should be atleast 2, however "
+        + std::to_string(neighborSize) + " was detected."
+    );
+  }
+
+  if (neighborSize > populationSize - 1u)
+  {
+    std::ostringstream oss;
+    oss << "MOEAD::Optimize(): " << "neighborSize is " << neighborSize
+        << " but populationSize is " << populationSize << "(should be"
+        << " atleast " << (neighborSize + 1u) << ")" << std::endl;
+    throw std::logic_error(oss.str());
+  }
+
+  // Check if lower bound is a vector of a single dimension.
+  if (lowerBound.n_rows == 1)
+    lowerBound = lowerBound(0, 0) * arma::ones(iterate.n_rows, iterate.n_cols);
+
+  // Check if upper bound is a vector of a single dimension.
+  if (upperBound.n_rows == 1)
+    upperBound = upperBound(0, 0) * arma::ones(iterate.n_rows, iterate.n_cols);
+
+  // Check the dimensions of lowerBound and upperBound.
+  assert(lowerBound.n_rows == iterate.n_rows && "The dimensions of "
+      "lowerBound are not the same as the dimensions of iterate.");
+  assert(upperBound.n_rows == iterate.n_rows && "The dimensions of "
+      "upperBound are not the same as the dimensions of iterate.");
+
+  const size_t numObjectives = sizeof...(ArbitraryFunctionType);
+  const size_t numVariables = iterate.n_rows;
+
+  //! Useful temporaries for float-like comparisons.
+  const BaseMatType castedLowerBound = arma::conv_to<BaseMatType>::from(lowerBound);
+  const BaseMatType castedUpperBound = arma::conv_to<BaseMatType>::from(upperBound);
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
+  // The weight matrix. Each vector represents a decomposition subproblem (M X N).
+  const BaseMatType weights = initPolicy.template Generate<BaseMatType>(
+      numObjectives, populationSize, epsilon);
+
+  // 1.1 Storing the indices of nearest neighbors of each weight vector.
+  arma::umat neighborIndices(neighborSize, populationSize);
+  for (size_t i = 0; i < populationSize; ++i)
+  {
+    // Cache the distance between weights[i] and other weights.
+    const arma::Row<ElemType> distances =
+        arma::sqrt(arma::sum(arma::square(weights.col(i) - weights.each_col())));
+    arma::uvec sortedIndices = arma::stable_sort_index(distances);
+    // Ignore distance from self.
+    neighborIndices.col(i) = sortedIndices(arma::span(1, neighborSize));
+  }
+
+  // 1.2 Random generation of the initial population.
+  std::vector<BaseMatType> population(populationSize);
+  for (BaseMatType& individual : population)
+  {
+    individual = arma::randu<BaseMatType>(
+        iterate.n_rows, iterate.n_cols) - 0.5 + iterate;
+
+    // Constrain all genes to be within bounds.
+    individual = arma::min(arma::max(individual, castedLowerBound), castedUpperBound);
+  }
+
+  Info << "MOEA/D-DE initialized successfully. Optimization started." << std::endl;
+
+  std::vector<arma::Col<ElemType>> populationFitness(populationSize);
+  std::fill(populationFitness.begin(), populationFitness.end(),
+      arma::Col<ElemType>(numObjectives, arma::fill::zeros));
+  EvaluateObjectives(population, objectives, populationFitness);
+
+  // 1.3 Initialize the ideal point z.
+  arma::Col<ElemType> idealPoint(numObjectives);
+  idealPoint.fill(std::numeric_limits<ElemType>::max());
+
+  for (const arma::Col<ElemType>& individualFitness : populationFitness)
+    idealPoint = arma::min(idealPoint, individualFitness);
+
+  terminate |= Callback::BeginOptimization(*this, objectives, iterate, callbacks...);
+
+  // 2 The main loop.
+  for (size_t generation = 1; generation <= maxGenerations && !terminate; ++generation)
+  {
+    // Shuffle indexes of subproblems.
+    const arma::uvec shuffle = arma::shuffle(
+        arma::linspace<arma::uvec>(0, populationSize - 1, populationSize));
+    for (size_t subProblemIdx : shuffle)
+    {
+      // 2.1 Randomly select two indices in neighborIndices[subProblemIdx] and use them
+      // to make a child.
+      size_t r1, r2, r3;
+      r1 = subProblemIdx;
+      // Randomly choose to sample from the population or the neighbors.
+      const bool sampleNeighbor = arma::randu() < neighborProb;
+      std::tie(r2, r3) =
+          Mating(subProblemIdx, neighborIndices, sampleNeighbor);
+
+      // 2.2 - 2.3 Reproduction and Repair: Differential Operator followed by
+      // Polynomial Mutation.
+      BaseMatType candidate(iterate.n_rows, iterate.n_cols);
+
+      for (size_t geneIdx = 0; geneIdx < numVariables; ++geneIdx)
+      {
+        if (arma::randu() < crossoverProb)
+        {
+          candidate(geneIdx) = population[r1](geneIdx) +
+              differentialWeight * (population[r2](geneIdx) -
+                  population[r3](geneIdx));
+
+          // Boundary conditions.
+          if (candidate(geneIdx) < castedLowerBound(geneIdx))
+          {
+            candidate(geneIdx) = castedLowerBound(geneIdx) +
+                arma::randu() * (population[r1](geneIdx) - castedLowerBound(geneIdx));
+          }
+          if (candidate(geneIdx) > castedUpperBound(geneIdx))
+          {
+            candidate(geneIdx) = castedUpperBound(geneIdx) -
+                arma::randu() * (castedUpperBound(geneIdx) - population[r1](geneIdx));
+          }
+        }
+        else
+          candidate(geneIdx) = population[r1](geneIdx);
+      }
+
+      Mutate(candidate, 1.0 / static_cast<double>(numVariables),
+          castedLowerBound, castedUpperBound);
+
+      arma::Col<ElemType> candidateFitness(numObjectives);
+      //! Creating temp vectors to pass to EvaluateObjectives.
+      std::vector<BaseMatType> candidateContainer { candidate };
+      std::vector<arma::Col<ElemType>> fitnessContainer { candidateFitness };
+      EvaluateObjectives(candidateContainer, objectives, fitnessContainer);
+      candidateFitness = std::move(fitnessContainer[0]);
+      //! Flush out the dummy containers.
+      fitnessContainer.clear();
+      candidateContainer.clear();
+
+      // 2.4 Update of ideal point.
+      idealPoint = arma::min(idealPoint, candidateFitness);
+
+      // 2.5 Update of the population.
+      size_t replaceCounter = 0;
+      const size_t sampleSize = sampleNeighbor ? neighborSize : populationSize;
+
+      const arma::uvec idxShuffle = arma::shuffle(
+          arma::linspace<arma::uvec>(0, sampleSize - 1, sampleSize));
+
+      for (size_t idx : idxShuffle)
+      {
+        // Preserve diversity by controlling replacement of neighbors
+        // by child solution.
+        if (replaceCounter >= maxReplace)
+          break;
+
+        const size_t pick = sampleNeighbor ?
+            neighborIndices(idx, subProblemIdx) : idx;
+
+        const ElemType candidateDecomposition = decompPolicy.template
+            Apply<arma::Col<ElemType>>(weights.col(pick), idealPoint, candidateFitness);
+        const ElemType parentDecomposition =  decompPolicy.template
+            Apply<arma::Col<ElemType>>(weights.col(pick), idealPoint, populationFitness[pick]);
+
+        if (candidateDecomposition < parentDecomposition)
+        {
+          population[pick] = candidate;
+          populationFitness[pick] = candidateFitness;
+          ++replaceCounter;
+        }
+      }
+    } // End of pass over all subproblems.
+
+    //  The final population itself is the best front.
+    const std::vector<arma::uvec> frontIndices { arma::shuffle(
+        arma::linspace<arma::uvec>(0, populationSize - 1, populationSize)) };
+
+    terminate |= Callback::GenerationalStepTaken(*this, objectives, iterate,
+        populationFitness, frontIndices, callbacks...);
+  } // End of pass over all the generations.
+
+  // Set the candidates from the Pareto Set as the output.
+  paretoSet.set_size(population[0].n_rows, population[0].n_cols, population.size());
+
+  // The Pareto Front is stored, can be obtained via ParetoSet() getter.
+  for (size_t solutionIdx = 0; solutionIdx < population.size(); ++solutionIdx)
+  {
+    paretoSet.slice(solutionIdx) =
+        arma::conv_to<arma::mat>::from(population[solutionIdx]);
+  }
+
+  // Set the candidates from the Pareto Front as the output.
+  paretoFront.set_size(populationFitness[0].n_rows, populationFitness[0].n_cols,
+      populationFitness.size());
+
+  // The Pareto Front is stored, can be obtained via ParetoFront() getter.
+  for (size_t solutionIdx = 0; solutionIdx < populationFitness.size(); ++solutionIdx)
+  {
+    paretoFront.slice(solutionIdx) =
+        arma::conv_to<arma::mat>::from(populationFitness[solutionIdx]);
+  }
+
+  // Assign iterate to first element of the Pareto Set.
+  iterate = population[0];
+
+  Callback::EndOptimization(*this, objectives, iterate, callbacks...);
+
+  ElemType performance = std::numeric_limits<ElemType>::max();
+
+  for (size_t geneIdx = 0; geneIdx < numObjectives; ++geneIdx)
+  {
+    if (arma::accu(populationFitness[geneIdx]) < performance)
+      performance = arma::accu(populationFitness[geneIdx]);
+  }
+
+  return performance;
+}
+
+//! Randomly chooses to select from parents or neighbors.
+template <typename InitPolicyType, typename DecompPolicyType>
+inline std::tuple<size_t, size_t>
+MOEAD<InitPolicyType, DecompPolicyType>::
+Mating(size_t subProblemIdx,
+       const arma::umat& neighborIndices,
+       bool sampleNeighbor)
+{
+  //! Indexes of two points from the sample space.
+  size_t pointA = sampleNeighbor
+      ? neighborIndices(
+            arma::randi(arma::distr_param(0, neighborSize - 1u)), subProblemIdx)
+      : arma::randi(arma::distr_param(0, populationSize - 1u));
+
+  size_t pointB = sampleNeighbor
+      ? neighborIndices(
+            arma::randi(arma::distr_param(0, neighborSize - 1u)), subProblemIdx)
+      : arma::randi(arma::distr_param(0, populationSize - 1u));
+
+  //! If the sampled points are equal, then modify one of them
+  //! within reasonable bounds.
+  if (pointA == pointB)
+  {
+    if (pointA == populationSize - 1u)
+      --pointA;
+    else
+      ++pointA;
+  }
+
+  return std::make_tuple(pointA, pointB);
+}
+
+//! Perform Polynomial mutation of the candidate.
+template <typename InitPolicyType, typename DecompPolicyType>
+template<typename MatType>
+inline void MOEAD<InitPolicyType, DecompPolicyType>::
+Mutate(MatType& candidate,
+       double mutationRate,
+       const MatType& lowerBound,
+       const MatType& upperBound)
+{
+    const size_t numVariables = candidate.n_rows;
+    for (size_t geneIdx = 0; geneIdx < numVariables; ++geneIdx)
+    {
+      // Should this gene be mutated?
+      if (arma::randu() > mutationRate)
+        continue;
+
+      const double geneRange = upperBound(geneIdx) - lowerBound(geneIdx);
+      // Normalised distance from the bounds.
+      const double lowerDelta = (candidate(geneIdx) - lowerBound(geneIdx)) / geneRange;
+      const double upperDelta = (upperBound(geneIdx) - candidate(geneIdx)) / geneRange;
+      const double mutationPower = 1. / (distributionIndex + 1.0);
+      const double rand = arma::randu();
+      double value, perturbationFactor;
+      if (rand < 0.5)
+      {
+        value = 2.0 * rand + (1.0 - 2.0 * rand) *
+            std::pow(upperDelta, distributionIndex + 1.0);
+        perturbationFactor = std::pow(value, mutationPower) - 1.0;
+      }
+      else
+      {
+        value = 2.0 * (1.0 - rand) + 2.0 *(rand - 0.5) *
+            std::pow(lowerDelta, distributionIndex + 1.0);
+        perturbationFactor = 1.0 - std::pow(value, mutationPower);
+      }
+
+      candidate(geneIdx) += perturbationFactor * geneRange;
+    }
+    //! Enforce bounds.
+    candidate = arma::min(arma::max(candidate, lowerBound), upperBound);
+}
+
+//! No objectives to evaluate.
+template <typename InitPolicyType, typename DecompPolicyType>
+template<std::size_t I,
+         typename MatType,
+         typename ...ArbitraryFunctionType>
+typename std::enable_if<I == sizeof...(ArbitraryFunctionType), void>::type
+MOEAD<InitPolicyType, DecompPolicyType>::
+EvaluateObjectives(
+    std::vector<MatType>&,
+    std::tuple<ArbitraryFunctionType...>&,
+    std::vector<arma::Col<typename MatType::elem_type> >&)
+{
+ // Nothing to do here.
+}
+
+//! Evaluate the objectives for the entire population.
+template <typename InitPolicyType, typename DecompPolicyType>
+template<std::size_t I,
+         typename MatType,
+         typename ...ArbitraryFunctionType>
+typename std::enable_if<I < sizeof...(ArbitraryFunctionType), void>::type
+MOEAD<InitPolicyType, DecompPolicyType>::
+EvaluateObjectives(
+    std::vector<MatType>& population,
+    std::tuple<ArbitraryFunctionType...>& objectives,
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives)
+{
+  for (size_t i = 0; i < population.size(); i++)
+  {
+    calculatedObjectives[i](I) = std::get<I>(objectives).Evaluate(population[i]);
+    EvaluateObjectives<I+1, MatType, ArbitraryFunctionType...>(population, objectives,
+                                                               calculatedObjectives);
+  }
+}
+
+}  // namespace ens
+
+#endif

include/ensmallen_bits/moead/weight_init_policies/bbs_init.hpp

@@ -0,0 +1,76 @@
+/**
+ * @file bbs_init.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * The Bayesian Bootstrap (BBS) method of Weight Initialization.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOEAD_BBS_HPP
+#define ENSMALLEN_MOEAD_BBS_HPP
+
+namespace ens {
+
+/**
+ * The Bayesian Bootstrap method for initializing weights. Samples are randomly picked from uniform 
+ * distribution followed by sorting and finding adjacent difference. This gives you a list of 
+ * numbers which is guaranteed to sum up to 1.
+ *
+ * @code
+ * @article{rubin1981bayesian,
+ *   title={The bayesian bootstrap},
+ *   author={Rubin, Donald B},
+ *   journal={The annals of statistics},
+ *   pages={130--134},
+ *   year={1981},
+ * @endcode
+ *
+ */
+class BayesianBootstrap
+{
+ public:
+  /**
+   * Constructor for Bayesian Bootstrap policy.
+   */
+  BayesianBootstrap()
+  {
+    /* Nothing to do. */
+  }
+
+  /**
+   * Generate the reference direction matrix.
+   *
+   * @tparam MatType The type of the matrix used for constructing weights.
+   * @param numObjectives The dimensionality of objective space.
+   * @param numPoints The number of reference directions requested.
+   * @param epsilon Handle numerical stability after weight initialization.
+   */
+  template<typename MatType>
+  MatType Generate(const size_t numObjectives,
+                   const size_t numPoints,
+                   const double epsilon)
+  {
+      typedef typename MatType::elem_type ElemType;
+      typedef typename arma::Col<ElemType> VecType;
+
+      MatType weights(numObjectives, numPoints);
+      for (size_t pointIdx = 0; pointIdx < numPoints; ++pointIdx)
+      {
+        VecType referenceDirection(numObjectives + 1, arma::fill::randu);
+        referenceDirection(0) = 0;
+        referenceDirection(numObjectives) = 1;
+        referenceDirection = arma::sort(referenceDirection);
+        referenceDirection = arma::diff(referenceDirection);
+        weights.col(pointIdx) = std::move(referenceDirection) + epsilon;
+      }
+
+      return weights;
+  }
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/moead/weight_init_policies/dirichlet_init.hpp

@@ -0,0 +1,55 @@
+/**
+ * @file dirichlet_init.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * The Dirichlet method of Weight Initialization.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOEAD_DIRICHLET_HPP
+#define ENSMALLEN_MOEAD_DIRICHLET_HPP
+
+namespace ens {
+
+/**
+ * The Dirichlet method for initializing weights. Sampling a 
+ * Dirichlet distribution with parameters set to one returns 
+ * point lying on unit simplex with uniform distribution.
+ */
+class Dirichlet
+{
+ public:
+  /**
+   * Constructor for Dirichlet policy.
+   */
+  Dirichlet()
+  {
+    /* Nothing to do. */
+  }
+
+  /**
+   * Generate the reference direction matrix.
+   *
+   * @tparam MatType The type of the matrix used for constructing weights.
+   * @param numObjectives The dimensionality of objective space.
+   * @param numPoints The number of reference directions requested.
+   * @param epsilon Handle numerical stability after weight initialization.
+   */
+  template<typename MatType>
+  MatType Generate(const size_t numObjectives,
+                   const size_t numPoints,
+                   const double epsilon)
+  {
+    MatType weights = arma::randg<MatType>(numObjectives, numPoints,
+        arma::distr_param(1.0, 1.0)) + epsilon;
+    // Normalize each column.
+    return arma::normalise(weights, 1, 0);
+  }
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/moead/weight_init_policies/uniform_init.hpp

@@ -0,0 +1,208 @@
+/**
+ * @file uniform_init.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * The Uniform (Das Dennis) methodology of Weight Initialization.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOEAD_UNIFORM_HPP
+#define ENSMALLEN_MOEAD_UNIFORM_HPP
+
+namespace ens {
+
+/**
+ * The Uniform (Das Dennis) method for initializing weights. This algorithm guarantees
+ * that the distance between adjacent points would be uniform.
+ *
+ * For more information, see the following:
+ *
+ * @code
+ * article{zhang2007moea,
+ *   title={MOEA/D: A multiobjective evolutionary algorithm based on decomposition},
+ *   author={Zhang, Qingfu and Li, Hui},
+ *   journal={IEEE Transactions on evolutionary computation},
+ *   pages={712--731},
+ *   year={2007}
+ * @endcode
+ */
+class Uniform
+{
+ public:
+  /**
+   * Constructor for Uniform Weight Initializatoin Policy.
+   */
+  Uniform()
+  {
+    /* Nothing to do. */
+  }
+
+  /**
+   * Generate the reference direction matrix.
+   *
+   * @tparam MatType The type of the matrix used for constructing weights.
+   * @param numObjectives The dimensionality of objective space.
+   * @param numPoints The number of reference directions requested.
+   * @param epsilon Handle numerical stability after weight initialization.
+   */
+  template<typename MatType>
+  MatType Generate(size_t numObjectives,
+                   size_t numPoints,
+                   double epsilon)
+  {
+    size_t numPartitions  = FindNumParitions(numObjectives, numPoints);
+    size_t validNumPoints = FindNumUniformPoints(numObjectives, numPartitions);
+
+    //! The requested number of points is not matching any partition number.
+    if (numPoints != validNumPoints)
+    {
+      size_t nextValidNumPoints = FindNumUniformPoints(numObjectives, numPartitions + 1);
+      std::ostringstream oss;
+      oss << "DasDennis::Generate(): " << "The requested numPoints " << numPoints
+          << " cannot be generated uniformly.\n " << "Either choose numPoints as "
+          << validNumPoints << " (numPartition = " << numPartitions << ") or "
+          << "numPoints as " << nextValidNumPoints << " (numPartition = "
+          << numPartitions + 1 << ").";
+      throw std::logic_error(oss.str());
+    }
+
+    return DasDennis<MatType>(numObjectives, numPoints,
+        numPartitions, epsilon);
+  }
+
+ private:
+  /**
+   * Finds the number of points which can be sampled uniformly from a 
+   * unit simplex given the number of partitions.
+   */
+  size_t FindNumUniformPoints(const size_t numObjectives,
+                              const size_t numPartitions)
+  {
+    //! O(N) algorithm to calculate binomial coefficient.
+    //! Source: https://www.geeksforgeeks.org/space-and-time-efficient-binomial-coefficient/
+    auto BinomialCoefficient =
+        [](size_t n, size_t k) -> size_t
+        {
+          size_t retval = 1;
+          // Since, C(n, k) = C(n, n - k).
+          if (k > n - k)
+            k = n - k;
+
+          // [n * (n - 1) * .... * (n - k + 1)] / [k * (k - 1) * .... * 1].
+          for (size_t i = 0; i < k; ++i)
+          {
+            retval *= (n - i);
+            retval /= (i + 1);
+          }
+
+          return retval;
+        };
+    return BinomialCoefficient(numObjectives + numPartitions - 1, numPartitions);
+  }
+
+  /**
+   *  Calculates the appropriate number of partitions such that, the binomial
+   *  coefficient value is closest to the number of points requested.
+   */
+  size_t FindNumParitions(size_t numObjectives, size_t numPoints)
+  {
+    if (numObjectives == 1) return 0;
+    // Iteratively increase numPartitions so that the binomial coefficient
+    // comes near to numPoints;
+    size_t numPartitions {1};
+    size_t sampledNumPoints = FindNumUniformPoints(numPartitions,
+        numObjectives);
+    while (sampledNumPoints <= numPoints)
+    {
+       ++numPartitions;
+       sampledNumPoints = FindNumUniformPoints(numObjectives,
+          numPartitions);
+    }
+
+    return numPartitions - 1;
+  }
+
+  /**
+   * A helper function for DasDennis
+   */
+  template<typename AuxInfoStackType,
+           typename MatType>
+  void DasDennisHelper(AuxInfoStackType& progressStack,
+                       MatType& weights,
+                       const size_t numObjectives,
+                       const size_t numPoints,
+                       const size_t numPartitions,
+                       const double epsilon)
+  {
+    typedef typename MatType::elem_type ElemType;
+    typedef typename arma::Row<ElemType> RowType;
+
+    size_t counter = 0;
+    const ElemType delta = 1.0 / (ElemType)numPartitions;
+
+    while ((counter < numPoints) && !progressStack.empty())
+    {
+      MatType point{};
+      size_t beta{};
+      std::tie(point, beta) = progressStack.back();
+      progressStack.pop_back();
+
+      if (point.size() + 1 == numObjectives)
+      {
+        point.insert_rows(point.n_rows, RowType(1).fill(
+            delta * static_cast<ElemType>(beta)));
+        weights.col(counter) = point + epsilon;
+        ++counter;
+      }
+
+      else
+      {
+        for (size_t i = 0; i <= beta; ++i)
+        {
+          MatType pointClone(point);
+          pointClone.insert_rows(pointClone.n_rows, RowType(1).fill(
+              delta * static_cast<ElemType>(i)));
+          progressStack.push_back({pointClone, beta - i});
+        }
+      }
+    }
+  }
+
+  /**
+   * Generates the weight matrix after verifying the
+   * validity of the parameters.
+   */
+  template <typename MatType>
+  MatType DasDennis(const size_t numObjectives,
+                    const size_t numPoints,
+                    const size_t numPartitions,
+                    const double epsilon)
+  {
+    //! Holds auxillary information required for the helper function. 
+    //! Holds the current point and beta value.
+    using AuxContainer = std::pair<MatType, size_t>;
+
+    std::vector<AuxContainer> progressStack{};
+    //! Init the progress stack.
+    progressStack.push_back({{}, numPartitions});
+    MatType weights(numObjectives, numPoints);
+    weights.fill(arma::datum::nan);
+    DasDennisHelper<decltype(progressStack), MatType>(
+        progressStack,
+        weights,
+        numObjectives,
+        numPoints,
+        numPartitions,
+        epsilon);
+
+    return weights;
+  }
+
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/nsga2/nsga2.hpp

@@ -1,6 +1,7 @@
 /**
  * @file nsga2.hpp
  * @author Sayan Goswami
+ * @author Nanubala Gnana Sai
  *
  * NSGA-II is a multi-objective optimization algorithm, widely used in
  * many real-world applications. NSGA-II generates offsprings using
@@ -50,7 +51,8 @@ namespace ens {
  * see the documentation on function types included with this distribution or
  * on the ensmallen website.
  */
-class NSGA2 {
+class NSGA2
+{
  public:
   /**
    * Constructor for the NSGA2 optimizer.
@@ -168,9 +170,33 @@ class NSGA2 {
   //! Modify value of upperBound.
   arma::vec& UpperBound() { return upperBound; }
 
-  //! Retrieve the best front (the Pareto frontier).  This returns an empty vector until `Optimize()`
-  //! has been called.
-  const std::vector<arma::mat>& Front() const { return bestFront; }
+  //! Retrieve the Pareto optimal points in variable space. This returns an empty cube
+  //! until `Optimize()` has been called.
+  const arma::cube& ParetoSet() const { return paretoSet; }
+
+  //! Retrieve the best front (the Pareto frontier). This returns an empty cube until
+  //! `Optimize()` has been called.
+  const arma::cube& ParetoFront() const { return paretoFront; }
+
+  /**
+   * Retrieve the best front (the Pareto frontier).  This returns an empty
+   * vector until `Optimize()` has been called.  Note that this function is
+   * deprecated and will be removed in ensmallen 3.x!  Use `ParetoFront()`
+   * instead.
+   */
+  ens_deprecated const std::vector<arma::mat>& Front()
+  {
+    if (rcFront.size() == 0)
+    {
+      // Match the old return format.
+      for (size_t i = 0; i < paretoFront.n_slices; ++i)
+      {
+        rcFront.push_back(arma::mat(paretoFront.slice(i)));
+      }
+    }
+
+    return rcFront;
+  }
 
  private:
   /**
@@ -188,7 +214,7 @@ class NSGA2 {
   typename std::enable_if<I == sizeof...(ArbitraryFunctionType), void>::type
   EvaluateObjectives(std::vector<MatType>&,
                      std::tuple<ArbitraryFunctionType...>&,
-                     std::vector<arma::Col<double> >&);
+                     std::vector<arma::Col<typename MatType::elem_type> >&);
 
   template<std::size_t I = 0,
            typename MatType,
@@ -196,7 +222,8 @@ class NSGA2 {
   typename std::enable_if<I < sizeof...(ArbitraryFunctionType), void>::type
   EvaluateObjectives(std::vector<MatType>& population,
                      std::tuple<ArbitraryFunctionType...>& objectives,
-                     std::vector<arma::Col<double> >& calculatedObjectives);
+                     std::vector<arma::Col<typename MatType::elem_type> >&
+                     calculatedObjectives);
 
   /**
    * Reproduce candidates from the elite population to generate a new
@@ -210,8 +237,8 @@ class NSGA2 {
    */
   template<typename MatType>
   void BinaryTournamentSelection(std::vector<MatType>& population,
-                                 const arma::vec& lowerBound,
-                                 const arma::vec& upperBound);
+                                 const MatType& lowerBound,
+                                 const MatType& upperBound);
 
   /**
    * Crossover two parents to create a pair of new children.
@@ -239,8 +266,8 @@ class NSGA2 {
    */
   template<typename MatType>
   void Mutate(MatType& child,
-              const arma::vec& lowerBound,
-              const arma::vec& upperBound);
+              const MatType& lowerBound,
+              const MatType& upperBound);
 
   /**
    * Sort the candidate population using their domination count and the set of
@@ -281,11 +308,15 @@ class NSGA2 {
    * Assigns crowding distance metric for sorting.
    *
    * @param front The previously generated Pareto fronts.
-   * @param objectives The set of objectives.
-   * @param crowdingDistance The previously calculated objectives.
+   * @param calculatedObjectives The previously calculated objectives.
+   * @param crowdingDistance The crowding distance for each individual in
+   *    the population.
    */
-  void CrowdingDistanceAssignment(const std::vector<size_t>& front,
-                                  std::vector<double>& crowdingDistance);
+  template <typename MatType>
+  void CrowdingDistanceAssignment(
+      const std::vector<size_t>& front,
+      std::vector<arma::Col<typename MatType::elem_type>>& calculatedObjectives,
+      std::vector<typename MatType::elem_type>& crowdingDistance);
 
   /**
    * The operator used in the crowding distance based sorting.
@@ -299,13 +330,15 @@ class NSGA2 {
    * @param idxQ The index of the second cadidate from the elite population
    *     being sorted.
    * @param ranks The previously calculated ranks.
-   * @param crowdingDistance The previously calculated objectives.
+   * @param crowdingDistance The crowding distance for each individual in
+   *    the population.
    * @return true if the first candidate is preferred, otherwise, false.
    */
+  template<typename MatType>
   bool CrowdingOperator(size_t idxP,
                         size_t idxQ,
                         const std::vector<size_t>& ranks,
-                        const std::vector<double>& crowdingDistance);
+                        const std::vector<typename MatType::elem_type>& crowdingDistance);
 
   //! The number of objectives being optimised for.
   size_t numObjectives;
@@ -337,8 +370,18 @@ class NSGA2 {
   //! Upper bound of the initial swarm.
   arma::vec upperBound;
 
-  //! Best front, stored after Optimize() is called.
-  std::vector<arma::mat> bestFront;
+  //! The set of all the Pareto optimal points.
+  //! Stored after Optimize() is called.
+  arma::cube paretoSet;
+
+  //! The set of all the Pareto optimal objective vectors.
+  //! Stored after Optimize() is called.
+  arma::cube paretoFront;
+
+  //! A different representation of the Pareto front, for reverse compatibility
+  //! purposes.  This can be removed when ensmallen 3.x is released!  (Along
+  //! with `Front()`.)  This is only populated when `Front()` is called.
+  std::vector<arma::mat> rcFront;
 };
 
 } // namespace ens

include/ensmallen_bits/nsga2/nsga2_impl.hpp

@@ -1,6 +1,7 @@
 /**
  * @file nsga2_impl.hpp
  * @author Sayan Goswami
+ * @author Nanubala Gnana Sai
  *
  * Implementation of the NSGA-II algorithm. Used for multi-objective
  * optimization problems on arbitrary functions.
@@ -27,6 +28,8 @@ inline NSGA2::NSGA2(const size_t populationSize,
                     const double epsilon,
                     const arma::vec& lowerBound,
                     const arma::vec& upperBound) :
+    numObjectives(0),
+    numVariables(0),
     populationSize(populationSize),
     maxGenerations(maxGenerations),
     crossoverProb(crossoverProb),
@@ -45,6 +48,8 @@ inline NSGA2::NSGA2(const size_t populationSize,
                     const double epsilon,
                     const double lowerBound,
                     const double upperBound) :
+    numObjectives(0),
+    numVariables(0),
     populationSize(populationSize),
     maxGenerations(maxGenerations),
     crossoverProb(crossoverProb),
@@ -61,7 +66,7 @@ template<typename MatType,
          typename... CallbackTypes>
 typename MatType::elem_type NSGA2::Optimize(
     std::tuple<ArbitraryFunctionType...>& objectives,
-    MatType& iterate,
+    MatType& iterateIn,
     CallbackTypes&&... callbacks)
 {
   // Make sure for evolution to work at least four candidates are present.
@@ -71,6 +76,17 @@ typename MatType::elem_type NSGA2::Optimize(
         " least 4, and, a multiple of 4!");
   }
 
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  // Make sure that we have the methods that we need.  Long name...
+  traits::CheckArbitraryFunctionTypeAPI<ArbitraryFunctionType...,
+      BaseMatType>();
+  RequireDenseFloatingPointType<BaseMatType>();
+
   // Check if lower bound is a vector of a single dimension.
   if (lowerBound.n_rows == 1)
     lowerBound = lowerBound(0, 0) * arma::ones(iterate.n_rows, iterate.n_cols);
@@ -85,29 +101,29 @@ typename MatType::elem_type NSGA2::Optimize(
   assert(upperBound.n_rows == iterate.n_rows && "The dimensions of "
       "upperBound are not the same as the dimensions of iterate.");
 
-  // Convenience typedefs.
-  typedef typename MatType::elem_type ElemType;
-
   numObjectives = sizeof...(ArbitraryFunctionType);
   numVariables = iterate.n_rows;
 
   // Cache calculated objectives.
-  std::vector<arma::Col<ElemType> > calculatedObjectives;
-  // Pre-allocate space for the calculated objectives.
-  calculatedObjectives.resize(populationSize);
+  std::vector<arma::Col<ElemType> > calculatedObjectives(populationSize);
 
   // Population size reserved to 2 * populationSize + 1 to accommodate
   // for the size of intermediate candidate population.
-  std::vector<MatType> population;
+  std::vector<BaseMatType> population;
   population.reserve(2 * populationSize + 1);
 
   // Pareto fronts, initialized during non-dominated sorting.
+  // Stores indices of population belonging to a certain front.
   std::vector<std::vector<size_t> > fronts;
   // Initialised in CrowdingDistanceAssignment.
-  std::vector<double> crowdingDistance;
+  std::vector<ElemType> crowdingDistance;
   // Initialised during non-dominated sorting.
   std::vector<size_t> ranks;
 
+  //! Useful temporaries for float-like comparisons.
+  const BaseMatType castedLowerBound = arma::conv_to<BaseMatType>::from(lowerBound);
+  const BaseMatType castedUpperBound = arma::conv_to<BaseMatType>::from(upperBound);
+
   // Controls early termination of the optimization process.
   bool terminate = false;
 
@@ -115,87 +131,103 @@ typename MatType::elem_type NSGA2::Optimize(
   // starting point.
   for (size_t i = 0; i < populationSize; i++)
   {
-    population.push_back(arma::randu<MatType>(iterate.n_rows,
+    population.push_back(arma::randu<BaseMatType>(iterate.n_rows,
         iterate.n_cols) - 0.5 + iterate);
+
+    // Constrain all genes to be within bounds.
+    population[i] = arma::min(arma::max(population[i], castedLowerBound), castedUpperBound);
   }
 
   Info << "NSGA2 initialized successfully. Optimization started." << std::endl;
 
-  // Evaluate the fitness before optimization.
-  for (size_t i = 0; i < population.size(); i++)
-    calculatedObjectives[i] = arma::Col<ElemType>(numObjectives, arma::fill::zeros);
-  EvaluateObjectives(population, objectives, calculatedObjectives);
-
   // Iterate until maximum number of generations is obtained.
   terminate |= Callback::BeginOptimization(*this, objectives, iterate, callbacks...);
 
   for (size_t generation = 1; generation <= maxGenerations && !terminate; generation++)
   {
     Info << "NSGA2: iteration " << generation << "." << std::endl;
-    terminate |= Callback::StepTaken(*this, objectives, iterate, callbacks...);
 
     // Create new population of candidate from the present elite population.
     // Have P_t, generate G_t using P_t.
-    BinaryTournamentSelection(population, lowerBound, upperBound);
+    BinaryTournamentSelection(population, castedLowerBound, castedUpperBound);
 
     // Evaluate the objectives for the new population.
     calculatedObjectives.resize(population.size());
-    for (size_t i = 0; i < population.size(); i++)
-      calculatedObjectives[i] = arma::Col<ElemType>(numObjectives, arma::fill::zeros);
+    std::fill(calculatedObjectives.begin(), calculatedObjectives.end(),
+        arma::Col<ElemType>(numObjectives, arma::fill::zeros));
     EvaluateObjectives(population, objectives, calculatedObjectives);
 
     // Perform fast non dominated sort on P_t ∪ G_t.
     ranks.resize(population.size());
-    FastNonDominatedSort<MatType>(fronts, ranks, calculatedObjectives);
+    FastNonDominatedSort<BaseMatType>(fronts, ranks, calculatedObjectives);
 
     // Perform crowding distance assignment.
     crowdingDistance.resize(population.size());
-
+    std::fill(crowdingDistance.begin(), crowdingDistance.end(), 0.);
     for (size_t fNum = 0; fNum < fronts.size(); fNum++)
     {
-      CrowdingDistanceAssignment(fronts[fNum], crowdingDistance);
+      CrowdingDistanceAssignment<BaseMatType>(
+          fronts[fNum], calculatedObjectives, crowdingDistance);
     }
 
     // Sort based on crowding distance.
     std::sort(population.begin(), population.end(),
-              [this, ranks, crowdingDistance, population](MatType candidateP,
-                                                          MatType candidateQ)
-              {
-                size_t idxP, idxQ;
-                for (size_t i = 0; i < population.size(); i++)
-                {
-                  if (arma::approx_equal(population[i], candidateP, "absdiff", epsilon))
-                    idxP = i;
+      [this, ranks, crowdingDistance, population]
+        (BaseMatType candidateP, BaseMatType candidateQ)
+          {
+            size_t idxP{}, idxQ{};
+            for (size_t i = 0; i < population.size(); i++)
+            {
+              if (arma::approx_equal(population[i], candidateP, "absdiff", epsilon))
+                idxP = i;
 
-                  if (arma::approx_equal(population[i], candidateQ, "absdiff", epsilon))
-                    idxQ = i;
-                }
+              if (arma::approx_equal(population[i], candidateQ, "absdiff", epsilon))
+                idxQ = i;
+            }
 
-                return CrowdingOperator(idxP, idxQ, ranks, crowdingDistance);
-              }
+            return CrowdingOperator<BaseMatType>(idxP, idxQ, ranks, crowdingDistance);
+          }
     );
 
     // Yield a new population P_{t+1} of size populationSize.
+    // Discards unfit population from the R_{t} to yield P_{t+1}.
     population.resize(populationSize);
+
+    terminate |= Callback::GenerationalStepTaken(*this, objectives, iterate,
+        calculatedObjectives, fronts, callbacks...);
   }
 
-  // Set the candidates from the best front as the output.
-  std::vector<MatType> front;
+  // Set the candidates from the Pareto Set as the output.
+  paretoSet.set_size(population[0].n_rows, population[0].n_cols, fronts[0].size());
+  // The Pareto Set is stored, can be obtained via ParetoSet() getter.
+  for (size_t solutionIdx = 0; solutionIdx < fronts[0].size(); ++solutionIdx)
+  {
+    paretoSet.slice(solutionIdx) =
+      arma::conv_to<arma::mat>::from(population[fronts[0][solutionIdx]]);
+  }
 
-  for (size_t f: fronts[0])
-    front.push_back(population[f]);
+  // Set the candidates from the Pareto Front as the output.
+  paretoFront.set_size(calculatedObjectives[0].n_rows, calculatedObjectives[0].n_cols,
+      fronts[0].size());
+  // The Pareto Front is stored, can be obtained via ParetoFront() getter.
+  for (size_t solutionIdx = 0; solutionIdx < fronts[0].size(); ++solutionIdx)
+  {
+    paretoFront.slice(solutionIdx) =
+      arma::conv_to<arma::mat>::from(calculatedObjectives[fronts[0][solutionIdx]]);
+  }
 
-  // bestFront is stored, can be obtained by the Front() getter.
-  bestFront = front;
+  // Clear rcFront, in case it is later requested by the user for reverse
+  // compatibility reasons.
+  rcFront.clear();
 
-  // Assign iterate to first element of the best front.
-  iterate = bestFront[0];
+  // Assign iterate to first element of the Pareto Set.
+  iterate = population[fronts[0][0]];
 
   Callback::EndOptimization(*this, objectives, iterate, callbacks...);
 
   ElemType performance = std::numeric_limits<ElemType>::max();
 
-  for(arma::Col<ElemType> objective: calculatedObjectives)
+  for (const arma::Col<ElemType>& objective: calculatedObjectives)
     if (arma::accu(objective) < performance)
       performance = arma::accu(objective);
 
@@ -210,7 +242,7 @@ typename std::enable_if<I == sizeof...(ArbitraryFunctionType), void>::type
 NSGA2::EvaluateObjectives(
     std::vector<MatType>&,
     std::tuple<ArbitraryFunctionType...>&,
-    std::vector<arma::Col<double> >&)
+    std::vector<arma::Col<typename MatType::elem_type> >&)
 {
   // Nothing to do here.
 }
@@ -223,7 +255,7 @@ typename std::enable_if<I < sizeof...(ArbitraryFunctionType), void>::type
 NSGA2::EvaluateObjectives(
     std::vector<MatType>& population,
     std::tuple<ArbitraryFunctionType...>& objectives,
-    std::vector<arma::Col<double> >& calculatedObjectives)
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives)
 {
   for (size_t i = 0; i < populationSize; i++)
   {
@@ -236,8 +268,8 @@ NSGA2::EvaluateObjectives(
 //! Reproduce and generate new candidates.
 template<typename MatType>
 inline void NSGA2::BinaryTournamentSelection(std::vector<MatType>& population,
-                                             const arma::vec& lowerBound,
-                                             const arma::vec& upperBound)
+                                             const MatType& lowerBound,
+                                             const MatType& upperBound)
 {
   std::vector<MatType> children;
 
@@ -292,20 +324,14 @@ inline void NSGA2::Crossover(MatType& childA,
 //! Perform mutation of the candidates weights with some noise.
 template<typename MatType>
 inline void NSGA2::Mutate(MatType& child,
-                          const arma::vec& lowerBound,
-                          const arma::vec& upperBound)
+                          const MatType& lowerBound,
+                          const MatType& upperBound)
 {
   child += (arma::randu<MatType>(child.n_rows, child.n_cols) < mutationProb) %
       (mutationStrength * arma::randn<MatType>(child.n_rows, child.n_cols));
 
   // Constrain all genes to be between bounds.
-  for (size_t idx = 0; idx < numVariables; idx++)
-  {
-    if (child[idx] < lowerBound(idx))
-      child[idx] = lowerBound(idx);
-    else if (child[idx] > upperBound(idx))
-      child[idx] = upperBound(idx);
-  }
+  child = arma::min(arma::max(child, lowerBound), upperBound);
 }
 
 //! Sort population into Pareto fronts.
@@ -344,7 +370,7 @@ inline void NSGA2::FastNonDominatedSort(
 
   size_t i = 0;
 
-  while (fronts[i].size() > 0)
+  while (!fronts[i].empty())
   {
     std::vector<size_t> nextFront;
 
@@ -365,6 +391,8 @@ inline void NSGA2::FastNonDominatedSort(
     i++;
     fronts.push_back(nextFront);
   }
+  // Remove the empty final set.
+  fronts.pop_back();
 }
 
 //! Check if a candidate Pareto dominates another candidate.
@@ -393,37 +421,59 @@ inline bool NSGA2::Dominates(
 }
 
 //! Assign crowding distance to the population.
-inline void NSGA2::CrowdingDistanceAssignment(const std::vector<size_t>& front,
-                                              std::vector<double>& crowdingDistance)
+template <typename MatType>
+inline void NSGA2::CrowdingDistanceAssignment(
+	const std::vector<size_t>& front,
+	std::vector<arma::Col<typename MatType::elem_type>>& calculatedObjectives,
+	std::vector<typename MatType::elem_type>& crowdingDistance)
 {
-  if (front.size() > 0)
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+
+  size_t fSize = front.size();
+  // Stores the sorted indices of the fronts.
+  arma::uvec sortedIdx  = arma::regspace<arma::uvec>(0, 1, fSize - 1);
+
+  for (size_t m = 0; m < numObjectives; m++)
   {
-    for (size_t elem: front)
-      crowdingDistance[elem] = 0;
+    // Cache fValues of individuals for current objective.
+    arma::Col<ElemType> fValues(fSize);
+    std::transform(front.begin(), front.end(), fValues.begin(),
+      [&](const size_t& individual)
+        {
+          return calculatedObjectives[individual](m);
+        });
 
-    size_t fSize = front.size();
+    // Sort front indices by ascending fValues for current objective.
+    std::sort(sortedIdx.begin(), sortedIdx.end(),
+      [&](const size_t& frontIdxA, const size_t& frontIdxB)
+        {
+          return (fValues(frontIdxA) < fValues(frontIdxB));
+        });
 
-    for (size_t m = 0; m < numObjectives; m++)
+    crowdingDistance[front[sortedIdx(0)]] =
+        std::numeric_limits<ElemType>::max();
+    crowdingDistance[front[sortedIdx(fSize - 1)]] =
+        std::numeric_limits<ElemType>::max();
+    ElemType minFval = fValues(sortedIdx(0));
+    ElemType maxFval = fValues(sortedIdx(fSize - 1));
+    ElemType scale =
+        std::abs(maxFval - minFval) == 0. ? 1. : std::abs(maxFval - minFval);
+
+    for (size_t i = 1; i < fSize - 1; i++)
     {
-      crowdingDistance[front[0]] = std::numeric_limits<double>::max();
-      crowdingDistance[front[fSize - 1]] = std::numeric_limits<double>::max();
-
-      for (size_t i = 1; i < fSize - 1 ; i++)
-      {
-        crowdingDistance[front[i]] += (crowdingDistance[front[i - 1]] -
-            crowdingDistance[front[i + 1]]) /
-            (std::numeric_limits<double>::max() -
-            std::numeric_limits<double>::min());
-      }
+      crowdingDistance[front[sortedIdx(i)]] +=
+          (fValues(sortedIdx(i + 1)) - fValues(sortedIdx(i - 1))) / scale;
     }
   }
 }
 
 //! Comparator for crowding distance based sorting.
+template<typename MatType>
 inline bool NSGA2::CrowdingOperator(size_t idxP,
                                     size_t idxQ,
                                     const std::vector<size_t>& ranks,
-                                    const std::vector<double>& crowdingDistance)
+                                    const std::vector<typename MatType::elem_type>& crowdingDistance)
 {
   if (ranks[idxP] < ranks[idxQ])
     return true;

include/ensmallen_bits/padam/padam_update.hpp

@@ -50,8 +50,7 @@ class PadamUpdate
       epsilon(epsilon),
       beta1(beta1),
       beta2(beta2),
-      partial(partial),
-      iteration(0)
+      partial(partial)
   {
     // Nothing to do.
   }
@@ -76,11 +75,6 @@ class PadamUpdate
   //! Modify the partial adaptive parameter.
   double& Partial() { return partial; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   /**
    * The UpdatePolicyType policy classes must contain an internal 'Policy'
    * template class with two template arguments: MatType and GradType.  This is
@@ -100,7 +94,8 @@ class PadamUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(PadamUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
@@ -119,7 +114,7 @@ class PadamUpdate
                 const GradType& gradient)
     {
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // And update the iterate.
       m *= parent.beta1;
@@ -128,10 +123,8 @@ class PadamUpdate
       v *= parent.beta2;
       v += (1 - parent.beta2) * (gradient % gradient);
 
-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
 
       // Element wise maximum of past and present squared gradients.
       vImproved = arma::max(vImproved, v);
@@ -152,6 +145,9 @@ class PadamUpdate
 
     //! The optimal sqaured gradient value.
     GradType vImproved;
+
+    //! The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -166,9 +162,6 @@ class PadamUpdate
 
   //! Partial adaptive parameter.
   double partial;
-
-  //! The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/problems/problems.hpp

@@ -42,5 +42,10 @@
 #include "styblinski_tang_function.hpp"
 #include "three_hump_camel_function.hpp"
 #include "wood_function.hpp"
+#include "zdt/zdt1_function.hpp"
+#include "zdt/zdt2_function.hpp"
+#include "zdt/zdt3_function.hpp"
+#include "zdt/zdt4_function.hpp"
+#include "zdt/zdt6_function.hpp"
 
 #endif

include/ensmallen_bits/problems/schwefel_function.hpp

@@ -114,7 +114,7 @@ class SchwefelFunction
   template<typename MatType = arma::mat>
   MatType GetFinalPoint() const
   {
-    MatType result(initialPoint.n_rows, initialPoint.n_cols);
+    MatType result(initialPoint.n_rows, initialPoint.n_cols, arma::fill::none);
     result.fill(420.9687);
     return result;
   }

include/ensmallen_bits/problems/zdt/zdt1_function.hpp

@@ -0,0 +1,157 @@
+/**
+ * @file zdt1_function.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * Implementation of the first ZDT(Zitzler, Deb and Thiele) test.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_ZDT_ONE_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_ZDT_ONE_FUNCTION_HPP
+
+namespace ens {
+namespace test {
+
+/**
+ * The ZDT1 function, defined by:
+ * \f[
+ * g(x) = 1 + 9(\sum_{i=2}^{n} x_i )/(n-1)
+ * f_1(x) = x_1
+ * h(f_1, g) = g(x)[1-\sqrt{f_1/g}\ ]
+ * f_2(x) = g(x) * h(f_1, g)
+ * \f]
+ *
+ * This is a 30-variable problem (n = 30) with a convex optimal set.
+ *
+ * Bounds of the variable space is:
+ * 0 <= x_i <= 1 for i = 1,...,n.
+ *
+ * This should be optimized to g(x) = 1.0, at:
+ * x_1* in [0, 1] ; x_i* = 0 for i = 2,...,n
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Zitzler2000,
+ *   title   = {Comparison of multiobjective evolutionary algorithms:
+ *              Empirical results},
+ *   author  = {Zitzler, Eckart and Deb, Kalyanmoy and Thiele, Lothar},
+ *   journal = {Evolutionary computation},
+ *   year    = {2000},
+ *   doi     = {10.1162/106365600568202}
+ * }
+ * @endcode
+ *
+ * @tparam MatType Type of matrix to optimize.
+ */
+  template<typename MatType = arma::mat>
+  class ZDT1
+  {
+   private:
+    size_t numParetoPoints {100};
+    size_t numObjectives {2};
+    size_t numVariables {30};
+
+   public:
+     //! Initialize the ZDT1
+    ZDT1(size_t numParetoPoints = 100) :
+        numParetoPoints(numParetoPoints),
+        objectiveF1(*this),
+        objectiveF2(*this)
+    {/* Nothing to do here. */}
+
+    /**
+     * Evaluate the objectives with the given coordinate.
+     *
+     * @param coords The function coordinates.
+     * @return arma::Col<typename MatType::elem_type>
+     */
+    arma::Col<typename MatType::elem_type> Evaluate(const MatType& coords)
+    {
+      // Convenience typedef.
+      typedef typename MatType::elem_type ElemType;
+
+      arma::Col<ElemType> objectives(numObjectives);
+      objectives(0) = coords[0];
+      ElemType sum = arma::accu(coords(arma::span(1, numVariables - 1), 0));
+      ElemType g = 1. + 9. * sum / (static_cast<ElemType>(numVariables) - 1.);
+      ElemType objectiveRatio = objectives(0) / g;
+      objectives(1) = g * (1. - std::sqrt(objectiveRatio));
+
+      return objectives;
+    }
+
+    //! Get the starting point.
+    MatType GetInitialPoint()
+    {
+      // Convenience typedef.
+      typedef typename MatType::elem_type ElemType;
+
+      return arma::Col<ElemType>(numVariables, 1, arma::fill::zeros);
+    }
+
+    struct ObjectiveF1
+    {
+      ObjectiveF1(ZDT1& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        return coords[0];
+      }
+
+      ZDT1& zdtClass;
+    };
+
+    struct ObjectiveF2
+    {
+      ObjectiveF2(ZDT1& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+
+        size_t numVariables = zdtClass.numVariables;
+        ElemType sum = arma::accu(coords(arma::span(1, numVariables - 1), 0));
+        ElemType g = 1. + 9. * sum / (static_cast<ElemType>(numVariables - 1));
+        ElemType objectiveRatio = zdtClass.objectiveF1.Evaluate(coords) / g;
+
+        return g * (1. - std::sqrt(objectiveRatio));
+      }
+
+      ZDT1& zdtClass;
+    };
+
+    //! Get objective functions.
+    std::tuple<ObjectiveF1, ObjectiveF2> GetObjectives()
+    {
+      return std::make_tuple(objectiveF1, objectiveF2);
+    }
+
+    //! Get the Reference Front.
+    //! Refer PR #273 Ipynb notebook to see the plot of Reference
+    //! Front. The implementation has been taken from pymoo.
+    arma::cube GetReferenceFront()
+    {
+      arma::cube front(2, 1, numParetoPoints);
+      arma::vec x = arma::linspace(0, 1, numParetoPoints);
+      arma::vec y = 1 - arma::sqrt(x);
+      for (size_t idx = 0; idx < numParetoPoints; ++idx)
+        front.slice(idx) = arma::vec{ x(idx), y(idx) };
+
+      return front;
+    }
+
+    ObjectiveF1 objectiveF1;
+    ObjectiveF2 objectiveF2;
+  };
+  } //namespace test
+  } //namespace ens
+
+#endif

include/ensmallen_bits/problems/zdt/zdt2_function.hpp

@@ -0,0 +1,158 @@
+/**
+ * @file zdt2_function.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * Implementation of the second ZDT(Zitzler, Deb and Thiele) test.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_ZDT_TWO_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_ZDT_TWO_FUNCTION_HPP
+
+namespace ens {
+namespace test {
+
+/**
+ * The ZDT2 function, defined by:
+ * \f[
+ * g(x) = 1 + 9(\sum_{i=2}^{n} x_i )/(n-1)
+ * f_1(x) = x_1
+ * h(f_1, g) = 1 - (f_1/g)^2
+ * f_2(x) = g(x) * h(f_1, g)
+ * \f]
+ *
+ * This is a 30-variable problem(n = 30) with a
+ * non convex optimal set.
+ *
+ * Bounds of the variable space is:
+ * 0 <= x_i <= 1 for i = 1,..,n.
+ *
+ * This should be optimized to g(x) = 1.0, at:
+ * x_1* in [0, 1] ; x_i* = 0 for i = 2,...,n
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Zitzler2000,
+ *   title   = {Comparison of multiobjective evolutionary algorithms:
+ *              Empirical results},
+ *   author  = {Zitzler, Eckart and Deb, Kalyanmoy and Thiele, Lothar},
+ *   journal = {Evolutionary computation},
+ *   year    = {2000},
+ *   doi     = {10.1162/106365600568202}
+ * }
+ * @endcode
+ *
+ * @tparam MatType Type of matrix to optimize.
+ */
+  template<typename MatType = arma::mat>
+  class ZDT2
+  {
+   private:
+    size_t numParetoPoints {100};
+    size_t numObjectives {2};
+    size_t numVariables {30};
+
+   public:
+     //! Initialize the ZDT2
+    ZDT2(size_t numParetoPoints = 100) :
+        numParetoPoints(numParetoPoints),
+        objectiveF1(*this),
+        objectiveF2(*this)
+    {/* Nothing to do here. */}
+
+    /**
+     * Evaluate the objectives with the given coordinate.
+     *
+     * @param coords The function coordinates.
+     * @return arma::Col<typename MatType::elem_type>
+     */
+    arma::Col<typename MatType::elem_type> Evaluate(const MatType& coords)
+    {
+      // Convenience typedef.
+      typedef typename MatType::elem_type ElemType;
+
+      arma::Col<ElemType> objectives(numObjectives);
+      objectives(0) = coords[0];
+      ElemType sum = arma::accu(coords(arma::span(1, numVariables - 1), 0));
+      ElemType g = 1. + 9. * sum / (static_cast<ElemType>(numVariables) - 1.);
+      ElemType objectiveRatio = objectives(0) / g;
+      objectives(1) = g * (1. - std::pow(objectiveRatio, 2));
+
+      return objectives;
+    }
+
+    //! Get the starting point.
+    MatType GetInitialPoint()
+    {
+      // Convenience typedef.
+      typedef typename MatType::elem_type ElemType;
+
+      return arma::Col<ElemType>(numVariables, 1, arma::fill::zeros);
+    }
+
+    struct ObjectiveF1
+    {
+      ObjectiveF1(ZDT2& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        return coords[0];
+      }
+
+      ZDT2& zdtClass;
+    };
+
+    struct ObjectiveF2
+    {
+      ObjectiveF2(ZDT2& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+
+        size_t numVariables = zdtClass.numVariables;
+        ElemType sum = arma::accu(coords(arma::span(1, numVariables - 1), 0));
+        ElemType g = 1. + 9. * sum / (static_cast<ElemType>(numVariables - 1));
+        ElemType objectiveRatio = zdtClass.objectiveF1.Evaluate(coords) / g;
+
+        return g * (1. - std::pow(objectiveRatio, 2));
+      }
+
+      ZDT2& zdtClass;
+    };
+
+    //! Get objective functions.
+    std::tuple<ObjectiveF1, ObjectiveF2> GetObjectives()
+    {
+      return std::make_tuple(objectiveF1, objectiveF2);
+    }
+
+    //! Get the Reference Front.
+    //! Refer PR #273 Ipynb notebook to see the plot of Reference
+    //! Front. The implementation has been taken from pymoo.
+    arma::cube GetReferenceFront()
+    {
+      arma::cube front(2, 1, numParetoPoints);
+      arma::vec x = arma::linspace(0, 1, numParetoPoints);
+      arma::vec y = 1 - arma::square(x);
+      for (size_t idx = 0; idx < numParetoPoints; ++idx)
+        front.slice(idx) = arma::vec{ x(idx), y(idx) };
+
+      return front;
+    }
+
+    ObjectiveF1 objectiveF1;
+    ObjectiveF2 objectiveF2;
+  };
+  } //namespace test
+  } //namespace ens
+
+#endif

include/ensmallen_bits/problems/zdt/zdt3_function.hpp

@@ -0,0 +1,188 @@
+/**
+ * @file zdt3_function.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * Implementation of the third ZDT(Zitzler, Deb and Thiele) test.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_ZDT_THREE_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_ZDT_THREE_FUNCTION_HPP
+
+namespace ens {
+namespace test {
+
+/**
+ * The ZDT3 function, defined by:
+ * \f[
+ * g(x) = 1 + 9(\sum_{i=2}^{n} x_i )/(n-1)
+ * f_1(x) = x_1
+ * h(f_1,g) = 1 - \sqrt{f_1/g}  - (f_1/g)sin(10\pi f_1)
+ * f_2(x) = g(x) * h(f_1, g)
+ * \f]
+ *
+ * This is a 30-variable problem(n = 30) with a number
+ * of disconnected optimal fronts.
+ *
+ * Bounds of the variable space is:
+ * 0 <= x_i <= 1 for i = 1,...,n.
+ *
+ * This should be optimized to g(x) = 1.0, at:
+ *
+ * x_1* in [0.0000, 0.0830] OR
+ * x_1* in [0.1822, 0.2577] OR
+ * x_1* in [0.4093, 0.4538] OR
+ * x_1* in [0.6183, 0.6525] OR
+ * x_1* in [0.8233, 0.8518].
+ *
+ * x_i* = 0 for i = 2,...,n.
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Zitzler2000,
+ *   title   = {Comparison of multiobjective evolutionary algorithms:
+ *              Empirical results},
+ *   author  = {Zitzler, Eckart and Deb, Kalyanmoy and Thiele, Lothar},
+ *   journal = {Evolutionary computation},
+ *   year    = {2000},
+ *   doi     = {10.1162/106365600568202}
+ * }
+ * @endcode
+ *
+ * @tparam MatType Type of matrix to optimize.
+ */
+  template<typename MatType = arma::mat>
+  class ZDT3
+  {
+   private:
+    size_t numParetoPoints {100};
+    size_t numObjectives {2};
+    size_t numVariables {30};
+
+   public:
+     //! Initialize the ZDT3
+    ZDT3(size_t numParetoPoints = 100) :
+        numParetoPoints(numParetoPoints),
+        objectiveF1(*this),
+        objectiveF2(*this)
+    {/* Nothing to do here. */}
+
+    /**
+     * Evaluate the objectives with the given coordinate.
+     *
+     * @param coords The function coordinates.
+     * @return arma::Col<typename MatType::elem_type>
+     */
+    arma::Col<typename MatType::elem_type> Evaluate(const MatType& coords)
+    {
+      typedef typename MatType::elem_type ElemType;
+
+      arma::Col<ElemType> objectives(numObjectives);
+      objectives(0) = coords[0];
+      ElemType sum = arma::accu(coords(arma::span(1, numVariables - 1), 0));
+      ElemType g = 1. + 9. * sum / (static_cast<ElemType>(numVariables) - 1.);
+      ElemType objectiveRatio = objectives(0) / g;
+      objectives(1) = g * (1. - std::sqrt(objectiveRatio) -
+          (objectiveRatio) * std::sin(10. * arma::datum::pi * coords[0]));
+
+      return objectives;
+    }
+
+    //! Get the starting point.
+    MatType GetInitialPoint()
+    {
+      // Convenience typedef.
+      typedef typename MatType::elem_type ElemType;
+
+      return arma::Col<ElemType>(numVariables, 1, arma::fill::zeros);
+    }
+
+    struct ObjectiveF1
+    {
+      ObjectiveF1(ZDT3& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        return coords[0];
+      }
+
+      ZDT3& zdtClass;
+    };
+
+    struct ObjectiveF2
+    {
+      ObjectiveF2(ZDT3& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        typedef typename MatType::elem_type ElemType;
+
+        size_t numVariables = zdtClass.numVariables;
+        ElemType sum = arma::accu(coords(arma::span(1, numVariables - 1), 0));
+        ElemType g = 1. + 9. * sum / (static_cast<ElemType>(numVariables - 1));
+        ElemType objectiveRatio = zdtClass.objectiveF1.Evaluate(coords) / g;
+
+        return g * (1. - std::sqrt(objectiveRatio) -
+            (objectiveRatio) * std::sin(10. * arma::datum::pi * coords[0]));
+      }
+
+      ZDT3& zdtClass;
+    };
+
+    //! Get objective functions.
+    std::tuple<ObjectiveF1, ObjectiveF2> GetObjectives()
+    {
+      return std::make_tuple(objectiveF1, objectiveF2);
+    }
+
+    //! Get the Reference Front.
+    //! Refer PR #273 Ipynb notebook to see the plot of Reference
+    //! Front. The implementation has been taken from pymoo.
+    arma::cube GetReferenceFront()
+    {
+      size_t numRegions = 5;
+      size_t regionDensity = std::floor(numParetoPoints / numRegions);
+      size_t apparentParetoPoints = numRegions * regionDensity;
+      arma::cube front(2, 1, apparentParetoPoints);
+
+      arma::mat regions{
+        {0.0, 0.182228780, 0.4093136748,
+         0.6183967944, 0.8233317983},
+        {0.0830015349, 0.2577623634, 0.4538821041,
+         0.6525117038, 0.8518328654}
+      };
+
+      for (size_t regionIdx = 0; regionIdx < numRegions; ++regionIdx)
+      {
+        arma::vec region = regions.col(regionIdx);
+        //! Generate x and y coordinates for the region.
+        arma::vec x = arma::linspace(
+            region(0), region(1), regionDensity);
+        arma::vec y = 1 - arma::sqrt(x) - x
+            % arma::sin(10 * arma::datum::pi * x);
+
+        //! Fill the front with the generated points.
+        for (size_t pointIdx = 0; pointIdx < regionDensity; ++pointIdx)
+        {
+          size_t sliceIdx = regionIdx * regionDensity + pointIdx;
+          front.slice(sliceIdx) = arma::vec{ x(pointIdx), y(pointIdx) };
+        }
+      }
+
+      return front;
+    }
+
+    ObjectiveF1 objectiveF1;
+    ObjectiveF2 objectiveF2;
+  };
+  } //namespace test
+  } //namespace ens
+
+#endif

include/ensmallen_bits/problems/zdt/zdt4_function.hpp

@@ -0,0 +1,160 @@
+/**
+ * @file zdt4_function.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * Implementation of the fourth ZDT(Zitzler, Deb and Thiele) test.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_ZDT_FOUR_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_ZDT_FOUR_FUNCTION_HPP
+
+namespace ens {
+namespace test {
+/**
+ * The ZDT4 function, defined by:
+ * \f[
+ * g(x) = 1 + 10(n-1) + \sum_{i=2}^{n}(x_i^2 - 10cos(4\pi x_i))
+ * f_1(x) = x_i
+ * h(f_1,g) = 1 - \sqrt{f_i/g}
+ * f_2(x) = g(x) * h(f_1, g)
+ * \f]
+ *
+ * This is a 10-variable problem(n = 10) with a convex
+ * optimal front. This problem contains several local
+ * optimum, making it difficult to reach the global optimum.
+ *
+ * Bounds of the variable space is:
+ *  0 <= x_1 <= 1;
+ * -10 <= x_i <= 10 for i = 2,...,n.
+ *
+ * This should be optimized to g(x) = 1.0, at:
+ * x_1* in [0, 1] ; x_i* = 0 for i = 2,...,n
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Zitzler2000,
+ *   title   = {Comparison of multiobjective evolutionary algorithms:
+ *              Empirical results},
+ *   author  = {Zitzler, Eckart and Deb, Kalyanmoy and Thiele, Lothar},
+ *   journal = {Evolutionary computation},
+ *   year    = {2000},
+ *   doi     = {10.1162/106365600568202}
+ * }
+ * @endcode
+ *
+ * @tparam MatType Type of matrix to optimize.
+ */
+  template<typename MatType = arma::mat>
+  class ZDT4
+  {
+   private:
+    size_t numParetoPoints {100};
+    size_t numObjectives {2};
+    size_t numVariables {10};
+
+   public:
+     //! Initialize the ZDT4
+    ZDT4(size_t numParetoPoints = 100) :
+        numParetoPoints(numParetoPoints),
+        objectiveF1(*this),
+        objectiveF2(*this)
+    {/* Nothing to do here. */}
+
+    /**
+     * Evaluate the objectives with the given coordinate.
+     *
+     * @param coords The function coordinates.
+     * @return arma::Col<typename MatType::elem_type>
+     */
+    arma::Col<typename MatType::elem_type> Evaluate(const MatType& coords)
+    {
+      typedef typename MatType::elem_type ElemType;
+
+      arma::Col<ElemType> objectives(numObjectives);
+      objectives(0) = coords[0];
+      MatType truncatedCoords = coords(arma::span(1, numVariables - 1), 0);
+      ElemType sum = arma::accu(arma::square(truncatedCoords) -
+          10. * arma::cos(4 * arma::datum::pi * truncatedCoords));
+      ElemType g = 1. + 10. * static_cast<ElemType>(numVariables - 1) + sum;
+      ElemType objectiveRatio = objectives(0) / g;
+	    objectives(1) = g * (1. - std::sqrt(objectiveRatio));
+
+      return objectives;
+    }
+
+    //! Get the starting point.
+    MatType GetInitialPoint()
+    {
+      // Convenience typedef.
+      typedef typename MatType::elem_type ElemType;
+
+      return arma::Col<ElemType>(numVariables, 1, arma::fill::zeros);
+    }
+
+    struct ObjectiveF1
+    {
+      ObjectiveF1(ZDT4& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        return coords[0];
+      }
+
+      ZDT4& zdtClass;
+    };
+
+    struct ObjectiveF2
+    {
+      ObjectiveF2(ZDT4& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        typedef typename MatType::elem_type ElemType;
+
+        size_t numVariables = zdtClass.numVariables;
+        MatType truncatedCoords = coords(arma::span(1, numVariables - 1), 0);
+        ElemType sum = arma::accu(arma::square(truncatedCoords) -
+            10. * arma::cos(4 * arma::datum::pi * truncatedCoords));
+        ElemType g = 1. + 10 * static_cast<ElemType>(numVariables - 1) + sum;
+        ElemType objectiveRatio = zdtClass.objectiveF1.Evaluate(coords) / g;
+
+        return  g * (1. - std::sqrt(objectiveRatio));
+      }
+
+      ZDT4& zdtClass;
+    };
+
+    //! Get objective functions.
+    std::tuple<ObjectiveF1, ObjectiveF2> GetObjectives()
+    {
+      return std::make_tuple(objectiveF1, objectiveF2);
+    }
+
+    //! Get the Reference Front.
+    //! Refer PR #273 Ipynb notebook to see the plot of Reference
+    //! Front. The implementation has been taken from pymoo.
+    arma::cube GetReferenceFront()
+    {
+      arma::cube front(2, 1, numParetoPoints);
+      arma::vec x = arma::linspace(0, 1, numParetoPoints);
+      arma::vec y = 1 - arma::sqrt(x);
+      for (size_t idx = 0; idx < numParetoPoints; ++idx)
+        front.slice(idx) = arma::vec{ x(idx), y(idx) };
+
+      return front;
+    }
+
+    ObjectiveF1 objectiveF1;
+    ObjectiveF2 objectiveF2;
+  };
+  } //namespace test
+  } //namespace ens
+#endif

include/ensmallen_bits/problems/zdt/zdt6_function.hpp

@@ -0,0 +1,162 @@
+/**
+ * @file zdt6_function.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * Implementation of the sixth ZDT(Zitzler, Deb and Thiele) test.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_ZDT_SIX_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_ZDT_SIX_FUNCTION_HPP
+
+namespace ens {
+namespace test {
+/**
+ * The ZDT6 function, defined by:
+ * \f[
+ * g(x) = 1 + 9[ \sum_{i=2}^{n}(x_i^2)/9]^{0.25}
+ * f_1(x) = 1 - e^{-4x_1}sin^{6}(6\pi x_1)
+ * h(f1, g) = 1 - (f_1/g)^{2}
+ * f_2(x) = g(x) * h(f_1, g)
+ * \f]
+ *
+ * This is a 10-variable problem(n = 10) with a
+ * non-convex optimal front. The density of the
+ * solutions across optimal region is non-uniform.
+ *
+ * Bounds of the variable space is:
+ * 0 <= x_i <= 1 for i = 1,...,n
+ *
+ * This should be optimized to g(x) = 1.0, at:
+ * x_1* in [0, 1] ; x_i* = 0 for i = 2,...,n
+ *
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Zitzler2000,
+ *   title   = {Comparison of multiobjective evolutionary algorithms:
+ *              Empirical results},
+ *   author  = {Zitzler, Eckart and Deb, Kalyanmoy and Thiele, Lothar},
+ *   journal = {Evolutionary computation},
+ *   year    = {2000},
+ *   doi     = {10.1162/106365600568202}
+ * }
+ * @endcode
+ *
+ * @tparam MatType Type of matrix to optimize.
+ */
+  template<typename MatType = arma::mat>
+  class ZDT6
+  {
+   private:
+    size_t numParetoPoints {100};
+    size_t numObjectives {2};
+    size_t numVariables {10};
+
+   public:
+     //! Initialize the ZDT6
+    ZDT6(size_t numParetoPoints = 100) :
+        numParetoPoints(numParetoPoints),
+        objectiveF1(*this),
+        objectiveF2(*this)
+    {/* Nothing to do here. */}
+
+    /**
+     * Evaluate the objectives with the given coordinate.
+     *
+     * @param coords The function coordinates.
+     * @return arma::Col<typename MatType::elem_type>
+     */
+    arma::Col<typename MatType::elem_type> Evaluate(const MatType& coords)
+    {
+      // Convenience typedef.
+      typedef typename MatType::elem_type ElemType;
+
+      arma::Col<ElemType> objectives(numObjectives);
+      objectives(0) = 1. - std::exp(-4 * coords[0]) *
+          std::pow(std::sin(6 * arma::datum::pi * coords[0]), 6);
+      ElemType sum = std::pow(
+          arma::accu(coords(arma::span(1, numVariables - 1), 0)) / 9, 0.25);
+      ElemType g = 1. + 9. * sum;
+      ElemType objectiveRatio = objectives(0) / g;
+      objectives(1) = g * (1. - std::pow(objectiveRatio, 2));
+
+      return objectives;
+    }
+
+    //! Get the starting point.
+    MatType GetInitialPoint()
+    {
+      // Convenience typedef.
+      typedef typename MatType::elem_type ElemType;
+
+      return arma::Col<ElemType>(numVariables, 1, arma::fill::zeros);
+    }
+
+    struct ObjectiveF1
+    {
+      ObjectiveF1(ZDT6& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        return 1. - std::exp(-4. * coords[0]) *
+            std::pow(std::sin(6. * arma::datum::pi * coords[0]), 6.);
+      }
+
+      ZDT6& zdtClass;
+    };
+
+    struct ObjectiveF2
+    {
+      ObjectiveF2(ZDT6& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        typedef typename MatType::elem_type ElemType;
+
+        size_t numVariables = zdtClass.numVariables;
+
+        ElemType sum = std::pow(
+            arma::accu(coords(arma::span(1, numVariables - 1), 0)) / 9, 0.25);
+        ElemType g = 1. + 9. * sum;
+        ElemType objectiveRatio = zdtClass.objectiveF1.Evaluate(coords) / g;
+
+        return  g * (1. - std::pow(objectiveRatio, 2));
+    }
+
+      ZDT6& zdtClass;
+    };
+
+    //! Get objective functions.
+    std::tuple<ObjectiveF1, ObjectiveF2> GetObjectives()
+    {
+      return std::make_tuple(objectiveF1, objectiveF2);
+    }
+
+    //! Get the Reference Front.
+    //! Refer PR #273 Ipynb notebook to see the plot of Reference
+    //! Front. The implementation has been taken from pymoo.
+    arma::cube GetReferenceFront()
+    {
+      arma::cube front(2, 1, numParetoPoints);
+      arma::vec x = arma::linspace(0.2807753191, 1, numParetoPoints);
+      arma::vec y = 1 - arma::square(x);
+      for (size_t idx = 0; idx < numParetoPoints; ++idx)
+        front.slice(idx) = arma::vec{ x(idx), y(idx) };
+
+      return front;
+    }
+
+    ObjectiveF1 objectiveF1;
+    ObjectiveF2 objectiveF2;
+  };
+  } //namespace test
+  } //namespace ens
+#endif

include/ensmallen_bits/pso/update_policies/lbest_update.hpp

@@ -73,7 +73,8 @@ class LBestUpdate
      *
      * @param parent Instantiated parent class.
      */
-     Policy(const LBestUpdate& /* parent */) { /* Do nothing. */ }
+     Policy(const LBestUpdate& /* parent */) : n(0)
+     { /* Do nothing. */ }
 
      /**
       * The Initialize method is called by PSO Optimizer method before the

include/ensmallen_bits/qhadam/qhadam_update.hpp

@@ -54,8 +54,7 @@ class QHAdamUpdate
     beta1(beta1),
     beta2(beta2),
     v1(v1),
-    v2(v2),
-    iteration(0)
+    v2(v2)
   {
     // Nothing to do.
   }
@@ -75,11 +74,6 @@ class QHAdamUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   //! Get the first quasi-hyperbolic term.
   double V1() const { return v1; }
   //! Modify the first quasi-hyperbolic term.
@@ -109,7 +103,8 @@ class QHAdamUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(QHAdamUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
@@ -127,7 +122,7 @@ class QHAdamUpdate
                 const GradType& gradient)
     {
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // And update the iterate.
       m *= parent.beta1;
@@ -136,10 +131,8 @@ class QHAdamUpdate
       v *= parent.beta2;
       v += (1 - parent.beta2) * (gradient % gradient);
 
-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
 
       GradType mDash = m / biasCorrection1;
       GradType vDash = v / biasCorrection2;
@@ -160,6 +153,9 @@ class QHAdamUpdate
 
     // The exponential moving average of squared gradient values.
     GradType v;
+
+    // The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -177,9 +173,6 @@ class QHAdamUpdate
 
   // The second quasi-hyperbolic term.
   double v2;
-
-  // The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/sa/sa_impl.hpp

@@ -70,13 +70,13 @@ typename MatType::elem_type SA<CoolingScheduleType>::Optimize(
   ElemType energy = function.Evaluate(iterate);
   Callback::Evaluate(*this, function, iterate, energy, callbacks...);
 
-  ElemType oldEnergy = energy;
+  ElemType oldEnergy;
 
   size_t idx = 0;
   size_t sweepCounter = 0;
 
   BaseMatType accept(rows, cols, arma::fill::zeros);
-  BaseMatType moveSize(rows, cols);
+  BaseMatType moveSize(rows, cols, arma::fill::none);
   moveSize.fill(initMoveCoef);
 
   terminate |= Callback::BeginOptimization(*this, function, iterate,

include/ensmallen_bits/sarah/sarah_impl.hpp

@@ -89,11 +89,6 @@ SARAHType<UpdatePolicyType>::Optimize(
   BaseGradType gradient0(iterate.n_rows, iterate.n_cols);
   BaseMatType iterate0;
 
-  // Find the number of batches.
-  size_t numBatches = numFunctions / batchSize;
-  if (numFunctions % batchSize != 0)
-    ++numBatches; // Capture last few.
-
   const size_t actualMaxIterations = (maxIterations == 0) ?
       std::numeric_limits<size_t>::max() : maxIterations;
   terminate |= Callback::BeginOptimization(*this, function, iterate,

include/ensmallen_bits/sdp/primal_dual_impl.hpp

@@ -162,8 +162,8 @@ SolveKKTSystem(const SparseConstraintType& aSparse,
         "solve KKT system.");
   }
 
-  MatType subTerm(aSparse.n_cols, 1);
-  subTerm.zeros();
+  MatType subTerm(aSparse.n_cols, 1, arma::fill::zeros);
+  
   if (aSparse.n_rows)
   {
     dySparse = dy(arma::span(0, aSparse.n_rows - 1), 0);
@@ -303,7 +303,7 @@ typename MatType::elem_type PrimalDualSolver::Optimize(
 
   eInvFaSparseT.set_size(n2bar, sdp.NumSparseConstraints());
   eInvFaDenseT.set_size(n2bar, sdp.NumDenseConstraints());
-  m.set_size(sdp.NumConstraints(), sdp.NumConstraints());
+  m.zeros(sdp.NumConstraints(), sdp.NumConstraints());
 
   // Controls early termination of the optimization process.
   bool terminate = false;

include/ensmallen_bits/sgdr/snapshot_ensembles.hpp

@@ -64,6 +64,7 @@ class SnapshotEnsembles
     constStepSize(stepSize),
     nextRestart(epochRestart),
     batchRestart(0),
+    epochBatches(0),
     epoch(0)
   {
     snapshotEpochs = 0;

include/ensmallen_bits/spalera_sgd/spalera_sgd_impl.hpp

@@ -178,7 +178,6 @@ SPALeRASGD<DecayPolicyType>::Optimize(
     i += effectiveBatchSize;
     currentFunction += effectiveBatchSize;
     overallObjective += currentObjective;
-    currentObjective /= effectiveBatchSize;
 
     // Is this iteration the start of a sequence?
     if ((currentFunction % numFunctions) == 0)

include/ensmallen_bits/spalera_sgd/spalera_stepsize.hpp

@@ -52,7 +52,8 @@ class SPALeRAStepsize
                   const double adaptRate = 3.10e-8) :
       alpha(alpha),
       epsilon(epsilon),
-      adaptRate(adaptRate)
+      adaptRate(adaptRate),
+      lambda(0)
   {
     /* Nothing to do here. */
   }

include/ensmallen_bits/swats/swats_update.hpp

@@ -50,7 +50,6 @@ class SWATSUpdate
     epsilon(epsilon),
     beta1(beta1),
     beta2(beta2),
-    iteration(0),
     phaseSGD(false),
     sgdRate(0),
     sgdLambda(0)
@@ -73,11 +72,6 @@ class SWATSUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   //! Get whether the current phase is SGD.
   bool PhaseSGD() const { return phaseSGD; }
   //! Modify whether the current phase is SGD.
@@ -111,7 +105,8 @@ class SWATSUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(SWATSUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
@@ -131,7 +126,7 @@ class SWATSUpdate
                 const GradType& gradient)
     {
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       if (parent.phaseSGD)
       {
@@ -150,10 +145,8 @@ class SWATSUpdate
       v *= parent.beta2;
       v += (1 - parent.beta2) * (gradient % gradient);
 
-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
 
       GradType delta = stepSize * m / biasCorrection1 /
           (arma::sqrt(v / biasCorrection2) + parent.epsilon);
@@ -167,8 +160,7 @@ class SWATSUpdate
             (1 - parent.beta2) * rate;
         parent.sgdRate = parent.sgdLambda / biasCorrection2;
 
-        if (std::abs(parent.sgdRate - rate) < parent.epsilon &&
-            parent.iteration > 1)
+        if (std::abs(parent.sgdRate - rate) < parent.epsilon && iteration > 1)
         {
           parent.phaseSGD = true;
           v.zeros();
@@ -188,6 +180,9 @@ class SWATSUpdate
 
     //! The exponential moving average of squared gradient values (SGD).
     GradType sgdV;
+
+    //! The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -200,9 +195,6 @@ class SWATSUpdate
   //! The second moment coefficient.
   double beta2;
 
-  //! The number of iterations.
-  size_t iteration;
-
   //! Wether to use the SGD or Adam update rule.
   bool phaseSGD;
 

include/ensmallen_bits/utility/indicators/epsilon.hpp

@@ -0,0 +1,92 @@
+/**
+ * @file epsilon.hpp
+ * @author Rahul Ganesh Prabhu
+ * @author Nanubala Gnana Sai
+ *
+ * Epsilon indicator
+ * A binary quality indicator that is capable of detecting whether one
+ * approximation set is better than another.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_INDICATORS_EPSILON_HPP
+#define ENSMALLEN_INDICATORS_EPSILON_HPP
+
+namespace ens {
+
+/**
+ * The epsilon indicator is one of the binary quality indicators that was proposed by
+ * Zitzler et. al.. The indicator originally calculates a weak dominance relation 
+ * between two approximation sets. It returns "epsilon" which is the factor by which 
+ * the given approximation set is worse than the reference front with respect to 
+ * all the objectives.
+ * 
+ * \f[ I_{\epsilon}(A,B) = \max_{z^2 \in B} \
+ *                \min_{z^1 \in A} \
+ *                \max_{1 \leq i \leq n} \ \frac{z^1_i}{z^2_i}\
+ *                \f]
+ *
+ * For more information, please see:
+ *
+ * @code
+ * @article{1197687,
+ *    author   = {E. Zitzler and L. Thiele and M. Laumanns and C. M. Fonseca and
+ *                V. G. da Fonseca},
+ *    title    = {Performance assessment of multiobjective optimizers: an
+ *                analysis and review},
+ *    journal  = {IEEE Transactions on Evolutionary Computation},
+ *    year     = {2003},
+ * }
+ * @endcode
+ */
+  class Epsilon
+  {
+   public:
+    /**
+     * Default constructor does nothing, but is required to satisfy the Indicator
+     * policy.
+     */
+    Epsilon() { }
+
+    /**
+     * Find the epsilon value of the front with respect to the given reference
+     * front.
+     *
+     * @tparam CubeType The cube data type of front.
+     * @param front The given approximation front.
+     * @param referenceFront The given reference front.
+     * @return The epsilon value of the front.
+     */
+    template<typename CubeType>
+    static typename CubeType::elem_type Evaluate(const CubeType& front,
+                                                 const CubeType& referenceFront)
+    {
+      // Convenience typedefs.
+      typedef typename CubeType::elem_type ElemType;
+      ElemType eps = 0;
+      for (size_t i = 0; i < referenceFront.n_slices; i++)
+      {
+        ElemType epsjMin = std::numeric_limits<ElemType>::max();
+        for (size_t j = 0; j < front.n_slices; j++)
+        {
+          arma::Mat<ElemType> frontRatio = front.slice(j) / referenceFront.slice(i);
+          frontRatio.replace(arma::datum::inf, -1.); // Handle zero division case.
+          ElemType epsj = frontRatio.max();
+          if (epsj < epsjMin)
+            epsjMin = epsj;
+        }
+        if (epsjMin > eps)
+          eps = epsjMin;
+      }
+
+      return eps;
+    }
+  };
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/utility/indicators/igd_plus.hpp

@@ -0,0 +1,94 @@
+/**
+ * @file igd_plus.hpp
+ * @author Rahul Ganesh Prabhu
+ * @author Nanubala Gnana Sai
+ *
+ * Inverse Generational Distance Plus (IGD+) indicator.
+ * The average distance from each reference point to its nearest solution.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_INDICATORS_IGD_PLUS_HPP
+#define ENSMALLEN_INDICATORS_IGD_PLUS_HPP
+
+namespace ens {
+
+/**
+ * The IGD indicator returns the average distance from each point in the reference
+ * front to the nearest point to it's solution. IGD+ is an improvement upon
+ * the IGD indicator, which fixes misleading results given by IGD in certain
+ * cases via a different distance metric:
+ *
+ * \f[ d^{+}(z,a) = \sqrt{\sum_{i = 1}^{n}( \max\{a_i - z_i, 0\})^2 \ } \
+ *    \f]
+ *
+ * For more information see:
+ *
+ * @code
+ * @article{10.1007/978-3-319-15892-1_8,
+ *    author   = {Ishibuchi, Hisao and Masuda, Hiroyuki and Tanigaki, Yuki
+ *                and Nojima, Yusuke},
+ *    title    = {Modified Distance Calculation in Generational Distance
+ *                and Inverted Generational Distance},
+ *    book     = {Evolutionary Multi-Criterion Optimization}
+ *    year     = {2015}
+ * }
+ * @endcode
+ */
+  class IGDPlus
+  {
+   public:
+    /**
+     * Default constructor does nothing, but is required to satisfy the Indicator
+     * policy.
+     */
+    IGDPlus() { }
+
+    /**
+     * Find the IGD+ value of the front with respect to the given reference
+     * front.
+     *
+     * @tparam CubeType The cube data type of front.
+     * @param front The given approximation front.
+     * @param referenceFront The given reference front.
+     * @return The IGD value of the front.
+     */
+    template<typename CubeType>
+    static typename CubeType::elem_type Evaluate(const CubeType& front,
+                                                 const CubeType& referenceFront)
+    {
+      // Convenience typedefs.
+      typedef typename CubeType::elem_type ElemType;
+      ElemType igd = 0;
+      for (size_t i = 0; i < referenceFront.n_slices; i++)
+      {
+        ElemType min = std::numeric_limits<ElemType>::max();
+        for (size_t j = 0; j < front.n_slices; j++)
+        {
+          ElemType dist = 0;
+          for (size_t k = 0; k < front.slice(j).n_rows; k++)
+          {
+            ElemType z = referenceFront(k, 0, i);
+            ElemType a = front(k, 0, j);
+            // Assuming minimization of all objectives.
+            dist += std::pow(std::max<ElemType>(a - z, 0), 2);
+          }
+          dist = std::sqrt(dist);
+          if (dist < min)
+            min = dist;
+        }
+        igd += min;
+      }
+      igd /= referenceFront.n_slices;
+
+      return igd;
+    }
+  };
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/yogi/yogi.hpp

@@ -0,0 +1,189 @@
+/**
+ * @file yogi.hpp
+ * @author Marcus Edel 
+ *
+ * Class wrapper for the Yogi update Policy. Yogi is based on Adam with more
+ * fine grained effective learning rate control.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_YOGI_YOGI_HPP
+#define ENSMALLEN_YOGI_YOGI_HPP
+
+#include <ensmallen_bits/sgd/sgd.hpp>
+#include "yogi_update.hpp"
+
+namespace ens {
+
+/**
+ * Yogi is an variation of Adam with more fine grained effective learning rate
+ * control.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @inproceedings{Zaheer2018,
+ *   author    = {Zaheer, Manzil and Reddi, Sashank J. and Sachan, Devendra
+ *                and Kale, Satyen and Kumar, Sanjiv},
+ *   title     = {Adaptive Methods for Nonconvex Optimization},
+ *   year      = {2018},
+ *   publisher = {Curran Associates Inc.},
+ *   booktitle = {Proceedings of the 32nd International Conference on Neural
+ *                Information Processing Systems},
+ *   pages     = {9815–9825},
+ *   series    = {NIPS'18}
+ * }
+ * @endcode
+ *
+ * Yogi can optimize differentiable separable functions. For more details,
+ * see the documentation on function types included with this distribution or
+ * on the ensmallen website.
+ */
+class Yogi 
+{
+ public:
+  /**
+   * Construct the Yogi optimizer with the given function and parameters.
+   * Yogi is sensitive to its paramters and hence a good hyper paramater
+   * selection is necessary as its default may not fit every case.
+   *
+   * The maximum number of iterations refers to the maximum number of
+   * points that are processed (i.e., one iteration equals one point; one
+   * iteration does not equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in a single step.
+   * @param beta1 Exponential decay rate for the first moment estimates.
+   * @param beta2 Exponential decay rate for the weighted infinity norm
+   *     estimates.
+   * @param epsilon Value used to initialise the mean squared gradient
+   *     parameter.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *     limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *     function is visited in linear order.
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *     call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
+   */
+  Yogi(const double stepSize = 0.001,
+       const size_t batchSize = 32,
+       const double beta1 = 0.9,
+       const double beta2 = 0.999,
+       const double epsilon = 1e-8,
+       const size_t maxIterations = 100000,
+       const double tolerance = 1e-5,
+       const bool shuffle = true,
+       const bool resetPolicy = true,
+       const bool exactObjective = false);
+
+  /**
+   * Optimize the given function using Yogi. The given starting point will be
+   * modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
+  {
+    return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return optimizer.UpdatePolicy().Beta1(); }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return optimizer.UpdatePolicy().Beta1(); }
+
+  //! Get the second moment coefficient.
+  double Beta2() const { return optimizer.UpdatePolicy().Beta2(); }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return optimizer.UpdatePolicy().Beta2(); }
+
+  //! Get the value used to initialise the mean squared gradient parameter.
+  double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
+  //! Modify the value used to initialise the mean squared gradient parameter.
+  double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
+  //! Get whether or not the update policy parameters are reset before
+  //! Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+  private:
+  //! The Stochastic Gradient Descent object with Yogi policy.
+  SGD<YogiUpdate> optimizer;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "yogi_impl.hpp"
+
+#endif

include/ensmallen_bits/yogi/yogi_impl.hpp

@@ -0,0 +1,44 @@
+/**
+ * @file yogi_impl.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of Yogi class wrapper.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_YOGI_YOGI_IMPL_HPP
+#define ENSMALLEN_YOGI_YOGI_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "yogi.hpp"
+
+namespace ens {
+
+inline Yogi::Yogi(
+    const double stepSize,
+    const size_t batchSize,
+    const double beta1,
+    const double beta2,
+    const double epsilon,
+    const size_t maxIterations,
+    const double tolerance,
+    const bool shuffle,
+    const bool resetPolicy,
+    const bool exactObjective) :
+    optimizer(stepSize,
+              batchSize,
+              maxIterations,
+              tolerance,
+              shuffle,
+              YogiUpdate(epsilon, beta1, beta2),
+              NoDecay(),
+              resetPolicy,
+              exactObjective)
+{ /* Nothing to do. */ }
+
+} // namespace ens
+
+ #endif

include/ensmallen_bits/yogi/yogi_update.hpp

@@ -0,0 +1,146 @@
+/**
+ * @file yogi_update.hpp
+ * @author Marcus Edel
+ *
+ * Implements the Yogi Optimizer. Yogi is a variant of Adam with more fine
+ * grained effective learning rate control.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_YOGI_YOGI_UPDATE_HPP
+#define ENSMALLEN_YOGI_YOGI_UPDATE_HPP
+
+namespace ens {
+
+/**
+ * Yogi builds upon the Adam update strategy but provides more fine grained
+ * effective learning rate control.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @inproceedings{Zaheer2018,
+ *   author    = {Zaheer, Manzil and Reddi, Sashank J. and Sachan, Devendra
+ *                and Kale, Satyen and Kumar, Sanjiv},
+ *   title     = {Adaptive Methods for Nonconvex Optimization},
+ *   year      = {2018},
+ *   publisher = {Curran Associates Inc.},
+ *   booktitle = {Proceedings of the 32nd International Conference on Neural
+ *                Information Processing Systems},
+ *   pages     = {9815–9825},
+ *   series    = {NIPS'18}
+ * }
+ * @endcode
+ */
+class YogiUpdate
+{
+ public:
+  /**
+   * Construct the Yogi update policy with the given parameters.
+   *
+   * @param epsilon The epsilon value used to initialise the squared gradient
+   *     parameter.
+   * @param beta1 The smoothing parameter.
+   * @param beta2 The second moment coefficient.
+   * @param v1 The first quasi-hyperbolic term.
+   * @param v1 The second quasi-hyperbolic term.
+   */
+  YogiUpdate(const double epsilon = 1e-8,
+             const double beta1 = 0.9,
+             const double beta2 = 0.999) :
+    epsilon(epsilon),
+    beta1(beta1),
+    beta2(beta2)
+  {
+    // Nothing to do.
+  }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return epsilon; }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return beta1; }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return beta1; }
+
+  //! Get the second moment coefficient.
+  double Beta2() const { return beta2; }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return beta2; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent YogiUpdate object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(YogiUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent)
+    {
+      m.zeros(rows, cols);
+      v.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for Yogi.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      const MatType gSquared = arma::square(gradient);
+      v -= (1 - parent.beta2) * arma::sign(v - gSquared) % gSquared;
+
+      // Now update the iterate.
+      iterate -= stepSize * m / (arma::sqrt(v) + parent.epsilon);
+    }
+
+   private:
+    //! Instantiated parent object.
+    YogiUpdate& parent;
+
+    //! The exponential moving average of gradient values.
+    GradType m;
+
+    // The exponential moving average of squared gradient values.
+    GradType v;
+  };
+
+ private:
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+
+  // The smoothing parameter.
+  double beta1;
+
+  // The second moment coefficient.
+  double beta2;
+};
+
+} // namespace ens
+
+#endif

tests/CMakeLists.txt

@@ -1,9 +1,11 @@
 # The tests that need to be compiled.
 set(ENSMALLEN_TESTS_SOURCES
     main.cpp
+    ada_belief_test.cpp
     ada_bound_test.cpp
     ada_delta_test.cpp
     ada_grad_test.cpp
+    ada_sqrt_test.cpp
     adam_test.cpp
     aug_lagrangian_test.cpp
     bigbatch_sgd_test.cpp
@@ -11,6 +13,8 @@ set(ENSMALLEN_TESTS_SOURCES
     cmaes_test.cpp
     cne_test.cpp
     de_test.cpp
+    demon_adam_test.cpp
+    demon_sgd_test.cpp
     eve_test.cpp
     frankwolfe_test.cpp
     ftml_test.cpp
@@ -23,6 +27,7 @@ set(ENSMALLEN_TESTS_SOURCES
     line_search_test.cpp
     lookahead_test.cpp
     lrsdp_test.cpp
+    moead_test.cpp
     momentum_sgd_test.cpp
     nesterov_momentum_sgd_test.cpp
     nsga2_test.cpp
@@ -44,10 +49,11 @@ set(ENSMALLEN_TESTS_SOURCES
     svrg_test.cpp
     swats_test.cpp
     wn_grad_test.cpp
+    yogi_test.cpp
 )
 
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
-add_executable(ensmallen_tests ${ENSMALLEN_TESTS_SOURCES})
+add_executable(ensmallen_tests EXCLUDE_FROM_ALL ${ENSMALLEN_TESTS_SOURCES})
 target_link_libraries(ensmallen_tests PRIVATE ensmallen)
 
 # Copy test data into place.
@@ -59,4 +65,4 @@ add_custom_command(TARGET ensmallen_tests
 
 enable_testing()
 add_test(NAME ensmallen_tests COMMAND ensmallen_tests
-WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+WORKING_DIRECTORY ${CMAKE_BINARY_DIR})

tests/ada_belief_test.cpp

@@ -0,0 +1,71 @@
+/**
+ * @file ada_belief_test.cpp
+ * @author Marcus Edel
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#include <ensmallen.hpp>
+#include "catch.hpp"
+#include "test_function_tools.hpp"
+
+using namespace ens;
+using namespace ens::test;
+
+/**
+ * Test the AdaBelief optimizer on the Sphere function.
+ */
+TEST_CASE("AdaBeliefSphereFunctionTest", "[AdaBeliefTest]")
+{
+  AdaBelief optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false);
+  FunctionTest<SphereFunction>(optimizer, 0.5, 0.1);
+}
+
+/**
+ * Test the AdaBelief optimizer on the Sphere function with arma::fmat.
+ */
+TEST_CASE("AdaBeliefSphereFunctionTestFMat", "[AdaBeliefTest]")
+{
+  AdaBelief optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false);
+  FunctionTest<SphereFunction, arma::fmat>(optimizer, 0.5, 0.1);
+}
+
+/**
+ * Test the AdaBelief optimizer on the McCormick function.
+ */
+TEST_CASE("AdaBeliefMcCormickFunctionTest", "[AdaBeliefTest]")
+{
+  AdaBelief optimizer(0.5, 1, 0.7, 0.999, 1e-8, 500000, 1e-5, false);
+  FunctionTest<McCormickFunction>(optimizer, 0.5, 0.1);
+}
+
+/**
+ * Run AdaBelief on logistic regression and make sure the results are
+ * acceptable.
+ */
+TEST_CASE("AdaBeliefLogisticRegressionTest", "[AdaBeliefTest]")
+{
+  AdaBelief optimizer;
+  LogisticRegressionFunctionTest(optimizer, 0.003, 0.006);
+}
+
+/**
+ * Run AdaBelief on logistic regression and make sure the results are
+ * acceptable, using arma::fmat.
+ */
+TEST_CASE("AdaBeliefLogisticRegressionFMatTest", "[AdaBeliefTest]")
+{
+  arma::fmat data, testData, shuffledData;
+  arma::Row<size_t> responses, testResponses, shuffledResponses;
+
+  LogisticRegressionTestData(data, testData, shuffledData,
+      responses, testResponses, shuffledResponses);
+  LogisticRegression<arma::fmat> lr(shuffledData, shuffledResponses, 0.5);
+
+  AdaBelief optimizer;
+  LogisticRegressionFunctionTest<arma::fmat>(optimizer, 0.003, 0.006);
+}
+

tests/ada_sqrt_test.cpp

@@ -0,0 +1,34 @@
+/**
+ * @file ada_sqrt_test.cpp
+ * @author Marcus Edel
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#include <ensmallen.hpp>
+#include "catch.hpp"
+#include "test_function_tools.hpp"
+
+using namespace ens;
+using namespace ens::test;
+
+/**
+ * Run AdaSqrt on logistic regression and make sure the results are acceptable.
+ */
+TEST_CASE("AdaSqrtLogisticRegressionTest", "[AdaSqrtTest]")
+{
+  AdaSqrt optimizer(0.01, 32, 1e-8, 5000000, 1e-9, true);
+  LogisticRegressionFunctionTest(optimizer, 0.003, 0.006);
+}
+
+/**
+ * Run AdaSqrt on logistic regression and make sure the results are acceptable.
+ */
+TEST_CASE("AdaSqrtLogisticRegressionTestFMat", "[AdaSqrtTest]")
+{
+  AdaSqrt optimizer(0.01, 32, 1e-8, 5000000, 1e-9, true);
+  LogisticRegressionFunctionTest<arma::fmat>(optimizer, 0.003, 0.006);
+}

tests/adam_test.cpp

@@ -322,3 +322,25 @@ TEST_CASE("AdamThreeHumpCamelFunctionTest", "[AdamTest]")
   Adam optimizer(0.001, 2, 0.7, 0.999, 1e-8, 500000, 1e-9, false);
   FunctionTest<ThreeHumpCamelFunction>(optimizer, 0.1, 0.01);
 }
+
+/**
+ * Test that multiple runs of the Adam optimizer result in the exact same
+ * result.  This specifically tests that the update policy is successfully
+ * reset at the start of each optimization.
+ */
+TEST_CASE("AdamResetPolicyTest", "[AdamTest]")
+{
+  Adam optimizer(0.5, 2, 0.7, 0.999, 1e-8, 5, 1e-3, false);
+  optimizer.ResetPolicy() = true;
+
+  SphereFunction f(2);
+
+  arma::mat coordinatesA = f.GetInitialPoint();
+  optimizer.Optimize(f, coordinatesA);
+
+  // A second run should produce the exact same results.
+  arma::mat coordinatesB = f.GetInitialPoint();
+  optimizer.Optimize(f, coordinatesB);
+
+  CheckMatrices(coordinatesA, coordinatesB);
+}

tests/callbacks_test.cpp

@@ -33,7 +33,8 @@ class CompleteCallbackTestFunction
       calledEndOptimization(false),
       calledEvaluateConstraint(false),
       calledGradientConstraint(false),
-      calledStepTaken(false)
+      calledStepTaken(false),
+      calledGenerationalStepTaken(false)
   { }
 
   template<typename OptimizerType, typename FunctionType, typename MatType>
@@ -106,6 +107,18 @@ class CompleteCallbackTestFunction
                  MatType& /* coordinates */)
   { calledStepTaken = true; }
 
+    template<typename OptimizerType,
+             typename FunctionType,
+             typename MatType,
+             typename ObjectivesVecType,
+             typename IndicesType>
+    void GenerationalStepTaken(OptimizerType& /* optimizer */,
+                               FunctionType& /* function */,
+                               MatType& /* coordinates */,
+                               ObjectivesVecType& /* objectives */,
+                               IndicesType& /* frontIndices */)
+    { calledGenerationalStepTaken = true; }
+
   bool calledEvaluate;
   bool calledGradient;
   bool calledBeginEpoch;
@@ -115,6 +128,7 @@ class CompleteCallbackTestFunction
   bool calledEvaluateConstraint;
   bool calledGradientConstraint;
   bool calledStepTaken;
+  bool calledGenerationalStepTaken;
 };
 
 template<typename OptimizerType>
@@ -162,7 +176,8 @@ void CallbacksFullMultiobjectiveFunctionTest(OptimizerType& optimizer,
                                              bool calledEndOptimization,
                                              bool calledEvaluateConstraint,
                                              bool calledGradientConstraint,
-                                             bool calledStepTaken)
+                                             bool calledStepTaken,
+                                             bool calledGenerationalStepTaken)
 {
   SchafferFunctionN1<arma::mat> SCH;
 
@@ -185,6 +200,7 @@ void CallbacksFullMultiobjectiveFunctionTest(OptimizerType& optimizer,
   REQUIRE(cb.calledEvaluateConstraint == calledEvaluateConstraint);
   REQUIRE(cb.calledGradientConstraint == calledGradientConstraint);
   REQUIRE(cb.calledStepTaken == calledStepTaken);
+  REQUIRE(cb.calledGenerationalStepTaken == calledGenerationalStepTaken);
 }
 
 template<typename OptimizerType>
@@ -380,7 +396,19 @@ TEST_CASE("NSGA2CallbacksFullFunctionTest", "[CallbackTest]")
   arma::vec upperBound = {1000};
   NSGA2 optimizer(20, 5000, 0.5, 0.5, 1e-3, 1e-6, lowerBound, upperBound);
   CallbacksFullMultiobjectiveFunctionTest(optimizer, false, false, false, false,
-      true, true, false, false, true);
+      true, true, false, false, false, true);
+}
+
+/**
+ * Make sure we invoke all callbacks (MOEA/D-DE).
+ */
+TEST_CASE("MOEADCallbacksFullFunctionTest", "[CallbackTest]")
+{
+  arma::vec lowerBound = {-1000};
+  arma::vec upperBound = {1000};
+  DefaultMOEAD optimizer(150, 300, 1.0, 0.9, 20, 20, 0.5, 2, 1E-10, lowerBound, upperBound);
+  CallbacksFullMultiobjectiveFunctionTest(optimizer, false, false, false, false,
+      true, true, false, false, false, true);
 }
 
 /**
@@ -755,3 +783,76 @@ TEST_CASE("ReportCallbackTest", "[CallbacksTest]")
   aug.Optimize(f3, coordinates, Report(0.1, stream));
   REQUIRE(stream.str().length() > 0);
 }
+
+/**
+ * Make sure the GradClipByNorm callback will clip the gradient.
+ */
+TEST_CASE("GradClipByNormCallbackTest", "[CallbacksTest]")
+{
+  SGDTestFunction f;
+  arma::mat coordinates = f.GetInitialPoint();
+
+  StandardSGD s(0.0003, 1, 10, 1e-9, true);
+
+  std::stringstream stream;
+  s.Optimize(f, coordinates, GradClipByNorm(0.5), Report(0.1, stream));
+
+  // We don't store the gradient during the optimization process, so we use the
+  // output of the Report callback function to check if the gradient is
+  // clipped.
+  std::string line;
+  bool gradientInfo = false;
+  double gradient = 1;
+  while (std::getline(stream, line, '\n'))
+  {
+    if (gradientInfo)
+    {
+      size_t iter;
+      double loss, lossChange, stepSize, totalTime;
+
+      std::stringstream stream(line);
+      stream >> iter >> loss >> lossChange >> gradient >> stepSize >> totalTime;
+      break;
+    }
+
+    gradientInfo = line.find("|gradient|") != std::string::npos;
+  }
+
+  REQUIRE(gradient == 0.5);
+}
+/**
+ * Make sure the GradClipByValue callback will clip the gradient.
+ */
+TEST_CASE("GradClipByValueCallbackTest", "[CallbacksTest]")
+{
+  SGDTestFunction f;
+  arma::mat coordinates = f.GetInitialPoint();
+
+  StandardSGD s(0.0003, 1, 10, 1e-9, true);
+
+  std::stringstream stream;
+  s.Optimize(f, coordinates, GradClipByValue(0, 0), Report(0.1, stream));
+
+  // We don't store the gradient during the optimization process, so we use the
+  // output of the Report callback function to check if the gradient is
+  // clipped.
+  std::string line;
+  bool gradientInfo = false;
+  double gradient = 1;
+  while (std::getline(stream, line, '\n'))
+  {
+    if (gradientInfo)
+    {
+      size_t iter;
+      double loss, lossChange, stepSize, totalTime;
+
+      std::stringstream stream(line);
+      stream >> iter >> loss >> lossChange >> gradient >> stepSize >> totalTime;
+      break;
+    }
+
+    gradientInfo = line.find("|gradient|") != std::string::npos;
+  }
+
+  REQUIRE(gradient == 0);
+}

tests/catch.hpp

@@ -1,9 +1,9 @@
 /*
- *  Catch v2.13.4
- *  Generated: 2020-12-29 14:48:00.116107
+ *  Catch v2.13.8
+ *  Generated: 2022-01-03 21:20:09.589503
  *  ----------------------------------------------------------
  *  This file has been merged from multiple headers. Please don't edit it directly
- *  Copyright (c) 2020 Two Blue Cubes Ltd. All rights reserved.
+ *  Copyright (c) 2022 Two Blue Cubes Ltd. All rights reserved.
  *
  *  Distributed under the Boost Software License, Version 1.0. (See accompanying
  *  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
@@ -15,7 +15,7 @@
 
 #define CATCH_VERSION_MAJOR 2
 #define CATCH_VERSION_MINOR 13
-#define CATCH_VERSION_PATCH 4
+#define CATCH_VERSION_PATCH 8
 
 #ifdef __clang__
 #    pragma clang system_header
@@ -66,13 +66,16 @@
 #if !defined(CATCH_CONFIG_IMPL_ONLY)
 // start catch_platform.h
 
+// See e.g.:
+// https://opensource.apple.com/source/CarbonHeaders/CarbonHeaders-18.1/TargetConditionals.h.auto.html
 #ifdef __APPLE__
-# include <TargetConditionals.h>
-# if TARGET_OS_OSX == 1
-#  define CATCH_PLATFORM_MAC
-# elif TARGET_OS_IPHONE == 1
-#  define CATCH_PLATFORM_IPHONE
-# endif
+#  include <TargetConditionals.h>
+#  if (defined(TARGET_OS_OSX) && TARGET_OS_OSX == 1) || \
+      (defined(TARGET_OS_MAC) && TARGET_OS_MAC == 1)
+#    define CATCH_PLATFORM_MAC
+#  elif (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE == 1)
+#    define CATCH_PLATFORM_IPHONE
+#  endif
 
 #elif defined(linux) || defined(__linux) || defined(__linux__)
 #  define CATCH_PLATFORM_LINUX
@@ -132,9 +135,9 @@ namespace Catch {
 
 #endif
 
-// We have to avoid both ICC and Clang, because they try to mask themselves
-// as gcc, and we want only GCC in this block
-#if defined(__GNUC__) && !defined(__clang__) && !defined(__ICC) && !defined(__CUDACC__)
+// Only GCC compiler should be used in this block, so other compilers trying to
+// mask themselves as GCC should be ignored.
+#if defined(__GNUC__) && !defined(__clang__) && !defined(__ICC) && !defined(__CUDACC__) && !defined(__LCC__)
 #    define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION _Pragma( "GCC diagnostic push" )
 #    define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION  _Pragma( "GCC diagnostic pop" )
 
@@ -237,9 +240,6 @@ namespace Catch {
 // Visual C++
 #if defined(_MSC_VER)
 
-#  define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION __pragma( warning(push) )
-#  define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION  __pragma( warning(pop) )
-
 // Universal Windows platform does not support SEH
 // Or console colours (or console at all...)
 #  if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP)
@@ -248,13 +248,18 @@ namespace Catch {
 #    define CATCH_INTERNAL_CONFIG_WINDOWS_SEH
 #  endif
 
+#  if !defined(__clang__) // Handle Clang masquerading for msvc
+
 // MSVC traditional preprocessor needs some workaround for __VA_ARGS__
 // _MSVC_TRADITIONAL == 0 means new conformant preprocessor
 // _MSVC_TRADITIONAL == 1 means old traditional non-conformant preprocessor
-#  if !defined(__clang__) // Handle Clang masquerading for msvc
 #    if !defined(_MSVC_TRADITIONAL) || (defined(_MSVC_TRADITIONAL) && _MSVC_TRADITIONAL)
 #      define CATCH_INTERNAL_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR
 #    endif // MSVC_TRADITIONAL
+
+// Only do this if we're not using clang on Windows, which uses `diagnostic push` & `diagnostic pop`
+#    define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION __pragma( warning(push) )
+#    define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION  __pragma( warning(pop) )
 #  endif // __clang__
 
 #endif // _MSC_VER
@@ -323,7 +328,7 @@ namespace Catch {
   // Check if byte is available and usable
   #  if __has_include(<cstddef>) && defined(CATCH_CPP17_OR_GREATER)
   #    include <cstddef>
-  #    if __cpp_lib_byte > 0
+  #    if defined(__cpp_lib_byte) && (__cpp_lib_byte > 0)
   #      define CATCH_INTERNAL_CONFIG_CPP17_BYTE
   #    endif
   #  endif // __has_include(<cstddef>) && defined(CATCH_CPP17_OR_GREATER)
@@ -1007,34 +1012,34 @@ struct AutoReg : NonCopyable {
 
     #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR
         #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION(Name, Tags, ...) \
-            INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, typename TestType, __VA_ARGS__ )
+            INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, typename TestType, __VA_ARGS__ )
     #else
         #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION(Name, Tags, ...) \
-            INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, typename TestType, __VA_ARGS__ ) )
+            INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, typename TestType, __VA_ARGS__ ) )
     #endif
 
     #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR
         #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_SIG_NO_REGISTRATION(Name, Tags, Signature, ...) \
-            INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, Signature, __VA_ARGS__ )
+            INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, Signature, __VA_ARGS__ )
     #else
         #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_SIG_NO_REGISTRATION(Name, Tags, Signature, ...) \
-            INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, Signature, __VA_ARGS__ ) )
+            INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, Signature, __VA_ARGS__ ) )
     #endif
 
     #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR
         #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION( ClassName, Name, Tags,... ) \
-            INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____C_L_A_S_S____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ) , ClassName, Name, Tags, typename T, __VA_ARGS__ )
+            INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_C_L_A_S_S_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ) , ClassName, Name, Tags, typename T, __VA_ARGS__ )
     #else
         #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION( ClassName, Name, Tags,... ) \
-            INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____C_L_A_S_S____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ) , ClassName, Name, Tags, typename T, __VA_ARGS__ ) )
+            INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_C_L_A_S_S_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ) , ClassName, Name, Tags, typename T, __VA_ARGS__ ) )
     #endif
 
     #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR
         #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_SIG_NO_REGISTRATION( ClassName, Name, Tags, Signature, ... ) \
-            INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____C_L_A_S_S____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ) , ClassName, Name, Tags, Signature, __VA_ARGS__ )
+            INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_C_L_A_S_S_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ) , ClassName, Name, Tags, Signature, __VA_ARGS__ )
     #else
         #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_SIG_NO_REGISTRATION( ClassName, Name, Tags, Signature, ... ) \
-            INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____C_L_A_S_S____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ) , ClassName, Name, Tags, Signature, __VA_ARGS__ ) )
+            INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_C_L_A_S_S_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ) , ClassName, Name, Tags, Signature, __VA_ARGS__ ) )
     #endif
 #endif
 
@@ -1047,7 +1052,7 @@ struct AutoReg : NonCopyable {
         CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION \
         static void TestName()
     #define INTERNAL_CATCH_TESTCASE( ... ) \
-        INTERNAL_CATCH_TESTCASE2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ), __VA_ARGS__ )
+        INTERNAL_CATCH_TESTCASE2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ ), __VA_ARGS__ )
 
     ///////////////////////////////////////////////////////////////////////////////
     #define INTERNAL_CATCH_METHOD_AS_TEST_CASE( QualifiedMethod, ... ) \
@@ -1069,7 +1074,7 @@ struct AutoReg : NonCopyable {
         CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION \
         void TestName::test()
     #define INTERNAL_CATCH_TEST_CASE_METHOD( ClassName, ... ) \
-        INTERNAL_CATCH_TEST_CASE_METHOD2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ), ClassName, __VA_ARGS__ )
+        INTERNAL_CATCH_TEST_CASE_METHOD2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ ), ClassName, __VA_ARGS__ )
 
     ///////////////////////////////////////////////////////////////////////////////
     #define INTERNAL_CATCH_REGISTER_TESTCASE( Function, ... ) \
@@ -1110,18 +1115,18 @@ struct AutoReg : NonCopyable {
 
 #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR
     #define INTERNAL_CATCH_TEMPLATE_TEST_CASE(Name, Tags, ...) \
-        INTERNAL_CATCH_TEMPLATE_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, typename TestType, __VA_ARGS__ )
+        INTERNAL_CATCH_TEMPLATE_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, typename TestType, __VA_ARGS__ )
 #else
     #define INTERNAL_CATCH_TEMPLATE_TEST_CASE(Name, Tags, ...) \
-        INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, typename TestType, __VA_ARGS__ ) )
+        INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, typename TestType, __VA_ARGS__ ) )
 #endif
 
 #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR
     #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_SIG(Name, Tags, Signature, ...) \
-        INTERNAL_CATCH_TEMPLATE_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, Signature, __VA_ARGS__ )
+        INTERNAL_CATCH_TEMPLATE_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, Signature, __VA_ARGS__ )
 #else
     #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_SIG(Name, Tags, Signature, ...) \
-        INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, Signature, __VA_ARGS__ ) )
+        INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, Signature, __VA_ARGS__ ) )
 #endif
 
     #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2(TestName, TestFuncName, Name, Tags, Signature, TmplTypes, TypesList) \
@@ -1159,18 +1164,18 @@ struct AutoReg : NonCopyable {
 
 #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR
     #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE(Name, Tags, ...)\
-        INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, typename T,__VA_ARGS__)
+        INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, typename T,__VA_ARGS__)
 #else
     #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE(Name, Tags, ...)\
-        INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, typename T, __VA_ARGS__ ) )
+        INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, typename T, __VA_ARGS__ ) )
 #endif
 
 #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR
     #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_SIG(Name, Tags, Signature, ...)\
-        INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, Signature, __VA_ARGS__)
+        INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, Signature, __VA_ARGS__)
 #else
     #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_SIG(Name, Tags, Signature, ...)\
-        INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, Signature, __VA_ARGS__ ) )
+        INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, Signature, __VA_ARGS__ ) )
 #endif
 
     #define INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_2(TestName, TestFunc, Name, Tags, TmplList)\
@@ -1201,7 +1206,7 @@ struct AutoReg : NonCopyable {
         static void TestFunc()
 
     #define INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE(Name, Tags, TmplList) \
-        INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, TmplList )
+        INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, TmplList )
 
     #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( TestNameClass, TestName, ClassName, Name, Tags, Signature, ... ) \
         CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \
@@ -1234,18 +1239,18 @@ struct AutoReg : NonCopyable {
 
 #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR
     #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD( ClassName, Name, Tags,... ) \
-        INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____C_L_A_S_S____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ) , ClassName, Name, Tags, typename T, __VA_ARGS__ )
+        INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_C_L_A_S_S_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ) , ClassName, Name, Tags, typename T, __VA_ARGS__ )
 #else
     #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD( ClassName, Name, Tags,... ) \
-        INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____C_L_A_S_S____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ) , ClassName, Name, Tags, typename T, __VA_ARGS__ ) )
+        INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_C_L_A_S_S_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ) , ClassName, Name, Tags, typename T, __VA_ARGS__ ) )
 #endif
 
 #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR
     #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_SIG( ClassName, Name, Tags, Signature, ... ) \
-        INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____C_L_A_S_S____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ) , ClassName, Name, Tags, Signature, __VA_ARGS__ )
+        INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_C_L_A_S_S_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ) , ClassName, Name, Tags, Signature, __VA_ARGS__ )
 #else
     #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_SIG( ClassName, Name, Tags, Signature, ... ) \
-        INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____C_L_A_S_S____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ) , ClassName, Name, Tags, Signature, __VA_ARGS__ ) )
+        INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_C_L_A_S_S_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ) , ClassName, Name, Tags, Signature, __VA_ARGS__ ) )
 #endif
 
     #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2(TestNameClass, TestName, ClassName, Name, Tags, Signature, TmplTypes, TypesList)\
@@ -1286,18 +1291,18 @@ struct AutoReg : NonCopyable {
 
 #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR
     #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD( ClassName, Name, Tags, ... )\
-        INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), ClassName, Name, Tags, typename T, __VA_ARGS__ )
+        INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), ClassName, Name, Tags, typename T, __VA_ARGS__ )
 #else
     #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD( ClassName, Name, Tags, ... )\
-        INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), ClassName, Name, Tags, typename T,__VA_ARGS__ ) )
+        INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), ClassName, Name, Tags, typename T,__VA_ARGS__ ) )
 #endif
 
 #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR
     #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_SIG( ClassName, Name, Tags, Signature, ... )\
-        INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), ClassName, Name, Tags, Signature, __VA_ARGS__ )
+        INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), ClassName, Name, Tags, Signature, __VA_ARGS__ )
 #else
     #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_SIG( ClassName, Name, Tags, Signature, ... )\
-        INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), ClassName, Name, Tags, Signature,__VA_ARGS__ ) )
+        INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), ClassName, Name, Tags, Signature,__VA_ARGS__ ) )
 #endif
 
     #define INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_METHOD_2( TestNameClass, TestName, ClassName, Name, Tags, TmplList) \
@@ -1331,7 +1336,7 @@ struct AutoReg : NonCopyable {
         void TestName<TestType>::test()
 
 #define INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_METHOD(ClassName, Name, Tags, TmplList) \
-        INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), ClassName, Name, Tags, TmplList )
+        INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), ClassName, Name, Tags, TmplList )
 
 // end catch_test_registry.h
 // start catch_capture.hpp
@@ -3088,7 +3093,7 @@ namespace Detail {
         Approx operator-() const;
 
         template <typename T, typename = typename std::enable_if<std::is_constructible<double, T>::value>::type>
-        Approx operator()( T const& value ) {
+        Approx operator()( T const& value ) const {
             Approx approx( static_cast<double>(value) );
             approx.m_epsilon = m_epsilon;
             approx.m_margin = m_margin;
@@ -4160,7 +4165,7 @@ namespace Generators {
             if (!m_predicate(m_generator.get())) {
                 // It might happen that there are no values that pass the
                 // filter. In that case we throw an exception.
-                auto has_initial_value = next();
+                auto has_initial_value = nextImpl();
                 if (!has_initial_value) {
                     Catch::throw_exception(GeneratorException("No valid value found in filtered generator"));
                 }
@@ -4172,6 +4177,11 @@ namespace Generators {
         }
 
         bool next() override {
+            return nextImpl();
+        }
+
+    private:
+        bool nextImpl() {
             bool success = m_generator.next();
             if (!success) {
                 return false;
@@ -5455,6 +5465,8 @@ namespace Catch {
 } // namespace Catch
 
 // end catch_outlier_classification.hpp
+
+#include <iterator>
 #endif // CATCH_CONFIG_ENABLE_BENCHMARKING
 
 #include <string>
@@ -6339,9 +6351,10 @@ namespace Catch {
 
         void writeTestCase(TestCaseNode const& testCaseNode);
 
-        void writeSection(std::string const& className,
-                          std::string const& rootName,
-                          SectionNode const& sectionNode);
+        void writeSection( std::string const& className,
+                           std::string const& rootName,
+                           SectionNode const& sectionNode,
+                           bool testOkToFail );
 
         void writeAssertions(SectionNode const& sectionNode);
         void writeAssertion(AssertionStats const& stats);
@@ -6876,7 +6889,7 @@ namespace Catch {
                     }
                     iters *= 2;
                 }
-                throw optimized_away_error{};
+                Catch::throw_exception(optimized_away_error{});
             }
         } // namespace Detail
     } // namespace Benchmark
@@ -6884,6 +6897,7 @@ namespace Catch {
 
 // end catch_run_for_at_least.hpp
 #include <algorithm>
+#include <iterator>
 
 namespace Catch {
     namespace Benchmark {
@@ -7054,8 +7068,8 @@ namespace Catch {
                 double b2 = bias - z1;
                 double a1 = a(b1);
                 double a2 = a(b2);
-                auto lo = std::max(cumn(a1), 0);
-                auto hi = std::min(cumn(a2), n - 1);
+                auto lo = (std::max)(cumn(a1), 0);
+                auto hi = (std::min)(cumn(a2), n - 1);
 
                 return { point, resample[lo], resample[hi], confidence_level };
             }
@@ -7124,7 +7138,9 @@ namespace Catch {
             }
             template <typename Clock>
             EnvironmentEstimate<FloatDuration<Clock>> estimate_clock_cost(FloatDuration<Clock> resolution) {
-                auto time_limit = std::min(resolution * clock_cost_estimation_tick_limit, FloatDuration<Clock>(clock_cost_estimation_time_limit));
+                auto time_limit = (std::min)(
+                    resolution * clock_cost_estimation_tick_limit,
+                    FloatDuration<Clock>(clock_cost_estimation_time_limit));
                 auto time_clock = [](int k) {
                     return Detail::measure<Clock>([k] {
                         for (int i = 0; i < k; ++i) {
@@ -7771,7 +7787,7 @@ namespace Catch {
                 double sb = stddev.point;
                 double mn = mean.point / n;
                 double mg_min = mn / 2.;
-                double sg = std::min(mg_min / 4., sb / std::sqrt(n));
+                double sg = (std::min)(mg_min / 4., sb / std::sqrt(n));
                 double sg2 = sg * sg;
                 double sb2 = sb * sb;
 
@@ -7790,7 +7806,7 @@ namespace Catch {
                     return (nc / n) * (sb2 - nc * sg2);
                 };
 
-                return std::min(var_out(1), var_out(std::min(c_max(0.), c_max(mg_min)))) / sb2;
+                return (std::min)(var_out(1), var_out((std::min)(c_max(0.), c_max(mg_min)))) / sb2;
             }
 
             bootstrap_analysis analyse_samples(double confidence_level, int n_resamples, std::vector<double>::iterator first, std::vector<double>::iterator last) {
@@ -7980,86 +7996,58 @@ namespace Catch {
 
 // start catch_fatal_condition.h
 
-// start catch_windows_h_proxy.h
-
-
-#if defined(CATCH_PLATFORM_WINDOWS)
-
-#if !defined(NOMINMAX) && !defined(CATCH_CONFIG_NO_NOMINMAX)
-#  define CATCH_DEFINED_NOMINMAX
-#  define NOMINMAX
-#endif
-#if !defined(WIN32_LEAN_AND_MEAN) && !defined(CATCH_CONFIG_NO_WIN32_LEAN_AND_MEAN)
-#  define CATCH_DEFINED_WIN32_LEAN_AND_MEAN
-#  define WIN32_LEAN_AND_MEAN
-#endif
-
-#ifdef __AFXDLL
-#include <AfxWin.h>
-#else
-#include <windows.h>
-#endif
-
-#ifdef CATCH_DEFINED_NOMINMAX
-#  undef NOMINMAX
-#endif
-#ifdef CATCH_DEFINED_WIN32_LEAN_AND_MEAN
-#  undef WIN32_LEAN_AND_MEAN
-#endif
-
-#endif // defined(CATCH_PLATFORM_WINDOWS)
-
-// end catch_windows_h_proxy.h
-#if defined( CATCH_CONFIG_WINDOWS_SEH )
+#include <cassert>
 
 namespace Catch {
 
-    struct FatalConditionHandler {
-
-        static LONG CALLBACK handleVectoredException(PEXCEPTION_POINTERS ExceptionInfo);
-        FatalConditionHandler();
-        static void reset();
-        ~FatalConditionHandler();
-
-    private:
-        static bool isSet;
-        static ULONG guaranteeSize;
-        static PVOID exceptionHandlerHandle;
-    };
-
-} // namespace Catch
-
-#elif defined ( CATCH_CONFIG_POSIX_SIGNALS )
-
-#include <signal.h>
-
-namespace Catch {
-
-    struct FatalConditionHandler {
-
-        static bool isSet;
-        static struct sigaction oldSigActions[];
-        static stack_t oldSigStack;
-        static char altStackMem[];
-
-        static void handleSignal( int sig );
+    // Wrapper for platform-specific fatal error (signals/SEH) handlers
+    //
+    // Tries to be cooperative with other handlers, and not step over
+    // other handlers. This means that unknown structured exceptions
+    // are passed on, previous signal handlers are called, and so on.
+    //
+    // Can only be instantiated once, and assumes that once a signal
+    // is caught, the binary will end up terminating. Thus, there
+    class FatalConditionHandler {
+        bool m_started = false;
 
+        // Install/disengage implementation for specific platform.
+        // Should be if-defed to work on current platform, can assume
+        // engage-disengage 1:1 pairing.
+        void engage_platform();
+        void disengage_platform();
+    public:
+        // Should also have platform-specific implementations as needed
         FatalConditionHandler();
         ~FatalConditionHandler();
-        static void reset();
+
+        void engage() {
+            assert(!m_started && "Handler cannot be installed twice.");
+            m_started = true;
+            engage_platform();
+        }
+
+        void disengage() {
+            assert(m_started && "Handler cannot be uninstalled without being installed first");
+            m_started = false;
+            disengage_platform();
+        }
     };
 
-} // namespace Catch
-
-#else
-
-namespace Catch {
-    struct FatalConditionHandler {
-        void reset();
+    //! Simple RAII guard for (dis)engaging the FatalConditionHandler
+    class FatalConditionHandlerGuard {
+        FatalConditionHandler* m_handler;
+    public:
+        FatalConditionHandlerGuard(FatalConditionHandler* handler):
+            m_handler(handler) {
+            m_handler->engage();
+        }
+        ~FatalConditionHandlerGuard() {
+            m_handler->disengage();
+        }
     };
-}
 
-#endif
+} // end namespace Catch
 
 // end catch_fatal_condition.h
 #include <string>
@@ -8185,6 +8173,7 @@ namespace Catch {
         std::vector<SectionEndInfo> m_unfinishedSections;
         std::vector<ITracker*> m_activeSections;
         TrackerContext m_trackerContext;
+        FatalConditionHandler m_fatalConditionhandler;
         bool m_lastAssertionPassed = false;
         bool m_shouldReportUnexpected = true;
         bool m_includeSuccessfulResults;
@@ -10057,6 +10046,36 @@ namespace Catch {
 }
 
 // end catch_errno_guard.h
+// start catch_windows_h_proxy.h
+
+
+#if defined(CATCH_PLATFORM_WINDOWS)
+
+#if !defined(NOMINMAX) && !defined(CATCH_CONFIG_NO_NOMINMAX)
+#  define CATCH_DEFINED_NOMINMAX
+#  define NOMINMAX
+#endif
+#if !defined(WIN32_LEAN_AND_MEAN) && !defined(CATCH_CONFIG_NO_WIN32_LEAN_AND_MEAN)
+#  define CATCH_DEFINED_WIN32_LEAN_AND_MEAN
+#  define WIN32_LEAN_AND_MEAN
+#endif
+
+#ifdef __AFXDLL
+#include <AfxWin.h>
+#else
+#include <windows.h>
+#endif
+
+#ifdef CATCH_DEFINED_NOMINMAX
+#  undef NOMINMAX
+#endif
+#ifdef CATCH_DEFINED_WIN32_LEAN_AND_MEAN
+#  undef WIN32_LEAN_AND_MEAN
+#endif
+
+#endif // defined(CATCH_PLATFORM_WINDOWS)
+
+// end catch_windows_h_proxy.h
 #include <sstream>
 
 namespace Catch {
@@ -10573,7 +10592,7 @@ namespace Catch {
             // Extracts the actual name part of an enum instance
             // In other words, it returns the Blue part of Bikeshed::Colour::Blue
             StringRef extractInstanceName(StringRef enumInstance) {
-                // Find last occurence of ":"
+                // Find last occurrence of ":"
                 size_t name_start = enumInstance.size();
                 while (name_start > 0 && enumInstance[name_start - 1] != ':') {
                     --name_start;
@@ -10735,25 +10754,47 @@ namespace Catch {
 // end catch_exception_translator_registry.cpp
 // start catch_fatal_condition.cpp
 
-#if defined(__GNUC__)
-#    pragma GCC diagnostic push
-#    pragma GCC diagnostic ignored "-Wmissing-field-initializers"
-#endif
+#include <algorithm>
+
+#if !defined( CATCH_CONFIG_WINDOWS_SEH ) && !defined( CATCH_CONFIG_POSIX_SIGNALS )
+
+namespace Catch {
+
+    // If neither SEH nor signal handling is required, the handler impls
+    // do not have to do anything, and can be empty.
+    void FatalConditionHandler::engage_platform() {}
+    void FatalConditionHandler::disengage_platform() {}
+    FatalConditionHandler::FatalConditionHandler() = default;
+    FatalConditionHandler::~FatalConditionHandler() = default;
+
+} // end namespace Catch
+
+#endif // !CATCH_CONFIG_WINDOWS_SEH && !CATCH_CONFIG_POSIX_SIGNALS
+
+#if defined( CATCH_CONFIG_WINDOWS_SEH ) && defined( CATCH_CONFIG_POSIX_SIGNALS )
+#error "Inconsistent configuration: Windows' SEH handling and POSIX signals cannot be enabled at the same time"
+#endif // CATCH_CONFIG_WINDOWS_SEH && CATCH_CONFIG_POSIX_SIGNALS
 
 #if defined( CATCH_CONFIG_WINDOWS_SEH ) || defined( CATCH_CONFIG_POSIX_SIGNALS )
 
 namespace {
-    // Report the error condition
+    //! Signals fatal error message to the run context
     void reportFatal( char const * const message ) {
         Catch::getCurrentContext().getResultCapture()->handleFatalErrorCondition( message );
     }
-}
 
-#endif // signals/SEH handling
+    //! Minimal size Catch2 needs for its own fatal error handling.
+    //! Picked anecdotally, so it might not be sufficient on all
+    //! platforms, and for all configurations.
+    constexpr std::size_t minStackSizeForErrors = 32 * 1024;
+} // end unnamed namespace
+
+#endif // CATCH_CONFIG_WINDOWS_SEH || CATCH_CONFIG_POSIX_SIGNALS
 
 #if defined( CATCH_CONFIG_WINDOWS_SEH )
 
 namespace Catch {
+
     struct SignalDefs { DWORD id; const char* name; };
 
     // There is no 1-1 mapping between signals and windows exceptions.
@@ -10766,7 +10807,7 @@ namespace Catch {
         { static_cast<DWORD>(EXCEPTION_INT_DIVIDE_BY_ZERO), "Divide by zero error" },
     };
 
-    LONG CALLBACK FatalConditionHandler::handleVectoredException(PEXCEPTION_POINTERS ExceptionInfo) {
+    static LONG CALLBACK handleVectoredException(PEXCEPTION_POINTERS ExceptionInfo) {
         for (auto const& def : signalDefs) {
             if (ExceptionInfo->ExceptionRecord->ExceptionCode == def.id) {
                 reportFatal(def.name);
@@ -10777,38 +10818,50 @@ namespace Catch {
         return EXCEPTION_CONTINUE_SEARCH;
     }
 
-    FatalConditionHandler::FatalConditionHandler() {
-        isSet = true;
-        // 32k seems enough for Catch to handle stack overflow,
-        // but the value was found experimentally, so there is no strong guarantee
-        guaranteeSize = 32 * 1024;
-        exceptionHandlerHandle = nullptr;
-        // Register as first handler in current chain
-        exceptionHandlerHandle = AddVectoredExceptionHandler(1, handleVectoredException);
-        // Pass in guarantee size to be filled
-        SetThreadStackGuarantee(&guaranteeSize);
-    }
+    // Since we do not support multiple instantiations, we put these
+    // into global variables and rely on cleaning them up in outlined
+    // constructors/destructors
+    static PVOID exceptionHandlerHandle = nullptr;
 
-    void FatalConditionHandler::reset() {
-        if (isSet) {
-            RemoveVectoredExceptionHandler(exceptionHandlerHandle);
-            SetThreadStackGuarantee(&guaranteeSize);
-            exceptionHandlerHandle = nullptr;
-            isSet = false;
+    // For MSVC, we reserve part of the stack memory for handling
+    // memory overflow structured exception.
+    FatalConditionHandler::FatalConditionHandler() {
+        ULONG guaranteeSize = static_cast<ULONG>(minStackSizeForErrors);
+        if (!SetThreadStackGuarantee(&guaranteeSize)) {
+            // We do not want to fully error out, because needing
+            // the stack reserve should be rare enough anyway.
+            Catch::cerr()
+                << "Failed to reserve piece of stack."
+                << " Stack overflows will not be reported successfully.";
         }
     }
 
-    FatalConditionHandler::~FatalConditionHandler() {
-        reset();
+    // We do not attempt to unset the stack guarantee, because
+    // Windows does not support lowering the stack size guarantee.
+    FatalConditionHandler::~FatalConditionHandler() = default;
+
+    void FatalConditionHandler::engage_platform() {
+        // Register as first handler in current chain
+        exceptionHandlerHandle = AddVectoredExceptionHandler(1, handleVectoredException);
+        if (!exceptionHandlerHandle) {
+            CATCH_RUNTIME_ERROR("Could not register vectored exception handler");
+        }
     }
 
-bool FatalConditionHandler::isSet = false;
-ULONG FatalConditionHandler::guaranteeSize = 0;
-PVOID FatalConditionHandler::exceptionHandlerHandle = nullptr;
+    void FatalConditionHandler::disengage_platform() {
+        if (!RemoveVectoredExceptionHandler(exceptionHandlerHandle)) {
+            CATCH_RUNTIME_ERROR("Could not unregister vectored exception handler");
+        }
+        exceptionHandlerHandle = nullptr;
+    }
 
-} // namespace Catch
+} // end namespace Catch
 
-#elif defined( CATCH_CONFIG_POSIX_SIGNALS )
+#endif // CATCH_CONFIG_WINDOWS_SEH
+
+#if defined( CATCH_CONFIG_POSIX_SIGNALS )
+
+#include <signal.h>
 
 namespace Catch {
 
@@ -10817,10 +10870,6 @@ namespace Catch {
         const char* name;
     };
 
-    // 32kb for the alternate stack seems to be sufficient. However, this value
-    // is experimentally determined, so that's not guaranteed.
-    static constexpr std::size_t sigStackSize = 32768 >= MINSIGSTKSZ ? 32768 : MINSIGSTKSZ;
-
     static SignalDefs signalDefs[] = {
         { SIGINT,  "SIGINT - Terminal interrupt signal" },
         { SIGILL,  "SIGILL - Illegal instruction signal" },
@@ -10830,7 +10879,32 @@ namespace Catch {
         { SIGABRT, "SIGABRT - Abort (abnormal termination) signal" }
     };
 
-    void FatalConditionHandler::handleSignal( int sig ) {
+// Older GCCs trigger -Wmissing-field-initializers for T foo = {}
+// which is zero initialization, but not explicit. We want to avoid
+// that.
+#if defined(__GNUC__)
+#    pragma GCC diagnostic push
+#    pragma GCC diagnostic ignored "-Wmissing-field-initializers"
+#endif
+
+    static char* altStackMem = nullptr;
+    static std::size_t altStackSize = 0;
+    static stack_t oldSigStack{};
+    static struct sigaction oldSigActions[sizeof(signalDefs) / sizeof(SignalDefs)]{};
+
+    static void restorePreviousSignalHandlers() {
+        // We set signal handlers back to the previous ones. Hopefully
+        // nobody overwrote them in the meantime, and doesn't expect
+        // their signal handlers to live past ours given that they
+        // installed them after ours..
+        for (std::size_t i = 0; i < sizeof(signalDefs) / sizeof(SignalDefs); ++i) {
+            sigaction(signalDefs[i].id, &oldSigActions[i], nullptr);
+        }
+        // Return the old stack
+        sigaltstack(&oldSigStack, nullptr);
+    }
+
+    static void handleSignal( int sig ) {
         char const * name = "<unknown signal>";
         for (auto const& def : signalDefs) {
             if (sig == def.id) {
@@ -10838,16 +10912,33 @@ namespace Catch {
                 break;
             }
         }
-        reset();
-        reportFatal(name);
+        // We need to restore previous signal handlers and let them do
+        // their thing, so that the users can have the debugger break
+        // when a signal is raised, and so on.
+        restorePreviousSignalHandlers();
+        reportFatal( name );
         raise( sig );
     }
 
     FatalConditionHandler::FatalConditionHandler() {
-        isSet = true;
+        assert(!altStackMem && "Cannot initialize POSIX signal handler when one already exists");
+        if (altStackSize == 0) {
+            altStackSize = std::max(static_cast<size_t>(SIGSTKSZ), minStackSizeForErrors);
+        }
+        altStackMem = new char[altStackSize]();
+    }
+
+    FatalConditionHandler::~FatalConditionHandler() {
+        delete[] altStackMem;
+        // We signal that another instance can be constructed by zeroing
+        // out the pointer.
+        altStackMem = nullptr;
+    }
+
+    void FatalConditionHandler::engage_platform() {
         stack_t sigStack;
         sigStack.ss_sp = altStackMem;
-        sigStack.ss_size = sigStackSize;
+        sigStack.ss_size = altStackSize;
         sigStack.ss_flags = 0;
         sigaltstack(&sigStack, &oldSigStack);
         struct sigaction sa = { };
@@ -10859,40 +10950,17 @@ namespace Catch {
         }
     }
 
-    FatalConditionHandler::~FatalConditionHandler() {
-        reset();
-    }
-
-    void FatalConditionHandler::reset() {
-        if( isSet ) {
-            // Set signals back to previous values -- hopefully nobody overwrote them in the meantime
-            for( std::size_t i = 0; i < sizeof(signalDefs)/sizeof(SignalDefs); ++i ) {
-                sigaction(signalDefs[i].id, &oldSigActions[i], nullptr);
-            }
-            // Return the old stack
-            sigaltstack(&oldSigStack, nullptr);
-            isSet = false;
-        }
-    }
-
-    bool FatalConditionHandler::isSet = false;
-    struct sigaction FatalConditionHandler::oldSigActions[sizeof(signalDefs)/sizeof(SignalDefs)] = {};
-    stack_t FatalConditionHandler::oldSigStack = {};
-    char FatalConditionHandler::altStackMem[sigStackSize] = {};
-
-} // namespace Catch
-
-#else
-
-namespace Catch {
-    void FatalConditionHandler::reset() {}
-}
-
-#endif // signals/SEH handling
-
 #if defined(__GNUC__)
 #    pragma GCC diagnostic pop
 #endif
+
+    void FatalConditionHandler::disengage_platform() {
+        restorePreviousSignalHandlers();
+    }
+
+} // end namespace Catch
+
+#endif // CATCH_CONFIG_POSIX_SIGNALS
 // end catch_fatal_condition.cpp
 // start catch_generators.cpp
 
@@ -11447,7 +11515,8 @@ namespace {
             return lhs == rhs;
         }
 
-        auto ulpDiff = std::abs(lc - rc);
+        // static cast as a workaround for IBM XLC
+        auto ulpDiff = std::abs(static_cast<FP>(lc - rc));
         return static_cast<uint64_t>(ulpDiff) <= maxUlpDiff;
     }
 
@@ -11621,7 +11690,6 @@ Floating::WithinRelMatcher WithinRel(float target) {
 
 } // namespace Matchers
 } // namespace Catch
-
 // end catch_matchers_floating.cpp
 // start catch_matchers_generic.cpp
 
@@ -12955,9 +13023,8 @@ namespace Catch {
     }
 
     void RunContext::invokeActiveTestCase() {
-        FatalConditionHandler fatalConditionHandler; // Handle signals
+        FatalConditionHandlerGuard _(&m_fatalConditionhandler);
         m_activeTestCase->invoke();
-        fatalConditionHandler.reset();
     }
 
     void RunContext::handleUnfinishedSections() {
@@ -15320,7 +15387,7 @@ namespace Catch {
     }
 
     Version const& libraryVersion() {
-        static Version version( 2, 13, 4, "", 0 );
+        static Version version( 2, 13, 8, "", 0 );
         return version;
     }
 
@@ -16733,6 +16800,7 @@ CATCH_REGISTER_REPORTER("console", ConsoleReporter)
 #include <sstream>
 #include <ctime>
 #include <algorithm>
+#include <iomanip>
 
 namespace Catch {
 
@@ -16760,7 +16828,7 @@ namespace Catch {
 #else
             std::strftime(timeStamp, timeStampSize, fmt, timeInfo);
 #endif
-            return std::string(timeStamp);
+            return std::string(timeStamp, timeStampSize-1);
         }
 
         std::string fileNameTag(const std::vector<std::string> &tags) {
@@ -16771,6 +16839,17 @@ namespace Catch {
                 return it->substr(1);
             return std::string();
         }
+
+        // Formats the duration in seconds to 3 decimal places.
+        // This is done because some genius defined Maven Surefire schema
+        // in a way that only accepts 3 decimal places, and tools like
+        // Jenkins use that schema for validation JUnit reporter output.
+        std::string formatDuration( double seconds ) {
+            ReusableStringStream rss;
+            rss << std::fixed << std::setprecision( 3 ) << seconds;
+            return rss.str();
+        }
+
     } // anonymous namespace
 
     JunitReporter::JunitReporter( ReporterConfig const& _config )
@@ -16840,7 +16919,7 @@ namespace Catch {
         if( m_config->showDurations() == ShowDurations::Never )
             xml.writeAttribute( "time", "" );
         else
-            xml.writeAttribute( "time", suiteTime );
+            xml.writeAttribute( "time", formatDuration( suiteTime ) );
         xml.writeAttribute( "timestamp", getCurrentTimestamp() );
 
         // Write properties if there are any
@@ -16885,12 +16964,13 @@ namespace Catch {
         if ( !m_config->name().empty() )
             className = m_config->name() + "." + className;
 
-        writeSection( className, "", rootSection );
+        writeSection( className, "", rootSection, stats.testInfo.okToFail() );
     }
 
-    void JunitReporter::writeSection(  std::string const& className,
-                        std::string const& rootName,
-                        SectionNode const& sectionNode ) {
+    void JunitReporter::writeSection( std::string const& className,
+                                      std::string const& rootName,
+                                      SectionNode const& sectionNode,
+                                      bool testOkToFail) {
         std::string name = trim( sectionNode.stats.sectionInfo.name );
         if( !rootName.empty() )
             name = rootName + '/' + name;
@@ -16907,13 +16987,18 @@ namespace Catch {
                 xml.writeAttribute( "classname", className );
                 xml.writeAttribute( "name", name );
             }
-            xml.writeAttribute( "time", ::Catch::Detail::stringify( sectionNode.stats.durationInSeconds ) );
+            xml.writeAttribute( "time", formatDuration( sectionNode.stats.durationInSeconds ) );
             // This is not ideal, but it should be enough to mimic gtest's
             // junit output.
             // Ideally the JUnit reporter would also handle `skipTest`
             // events and write those out appropriately.
             xml.writeAttribute( "status", "run" );
 
+            if (sectionNode.stats.assertions.failedButOk) {
+                xml.scopedElement("skipped")
+                    .writeAttribute("message", "TEST_CASE tagged with !mayfail");
+            }
+
             writeAssertions( sectionNode );
 
             if( !sectionNode.stdOut.empty() )
@@ -16923,9 +17008,9 @@ namespace Catch {
         }
         for( auto const& childNode : sectionNode.childSections )
             if( className.empty() )
-                writeSection( name, "", *childNode );
+                writeSection( name, "", *childNode, testOkToFail );
             else
-                writeSection( className, name, *childNode );
+                writeSection( className, name, *childNode, testOkToFail );
     }
 
     void JunitReporter::writeAssertions( SectionNode const& sectionNode ) {
@@ -17570,9 +17655,9 @@ int main (int argc, char * const argv[]) {
 
 #if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
 #define CATCH_BENCHMARK(...) \
-    INTERNAL_CATCH_BENCHMARK(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), INTERNAL_CATCH_GET_1_ARG(__VA_ARGS__,,), INTERNAL_CATCH_GET_2_ARG(__VA_ARGS__,,))
+    INTERNAL_CATCH_BENCHMARK(INTERNAL_CATCH_UNIQUE_NAME(C_A_T_C_H_B_E_N_C_H_), INTERNAL_CATCH_GET_1_ARG(__VA_ARGS__,,), INTERNAL_CATCH_GET_2_ARG(__VA_ARGS__,,))
 #define CATCH_BENCHMARK_ADVANCED(name) \
-    INTERNAL_CATCH_BENCHMARK_ADVANCED(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), name)
+    INTERNAL_CATCH_BENCHMARK_ADVANCED(INTERNAL_CATCH_UNIQUE_NAME(C_A_T_C_H_B_E_N_C_H_), name)
 #endif // CATCH_CONFIG_ENABLE_BENCHMARKING
 
 // If CATCH_CONFIG_PREFIX_ALL is not defined then the CATCH_ prefix is not required
@@ -17674,9 +17759,9 @@ int main (int argc, char * const argv[]) {
 
 #if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
 #define BENCHMARK(...) \
-    INTERNAL_CATCH_BENCHMARK(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), INTERNAL_CATCH_GET_1_ARG(__VA_ARGS__,,), INTERNAL_CATCH_GET_2_ARG(__VA_ARGS__,,))
+    INTERNAL_CATCH_BENCHMARK(INTERNAL_CATCH_UNIQUE_NAME(C_A_T_C_H_B_E_N_C_H_), INTERNAL_CATCH_GET_1_ARG(__VA_ARGS__,,), INTERNAL_CATCH_GET_2_ARG(__VA_ARGS__,,))
 #define BENCHMARK_ADVANCED(name) \
-    INTERNAL_CATCH_BENCHMARK_ADVANCED(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), name)
+    INTERNAL_CATCH_BENCHMARK_ADVANCED(INTERNAL_CATCH_UNIQUE_NAME(C_A_T_C_H_B_E_N_C_H_), name)
 #endif // CATCH_CONFIG_ENABLE_BENCHMARKING
 
 using Catch::Detail::Approx;
@@ -17723,8 +17808,8 @@ using Catch::Detail::Approx;
 #define CATCH_WARN( msg )          (void)(0)
 #define CATCH_CAPTURE( msg )       (void)(0)
 
-#define CATCH_TEST_CASE( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ))
-#define CATCH_TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ))
+#define CATCH_TEST_CASE( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ ))
+#define CATCH_TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ ))
 #define CATCH_METHOD_AS_TEST_CASE( method, ... )
 #define CATCH_REGISTER_TEST_CASE( Function, ... ) (void)(0)
 #define CATCH_SECTION( ... )
@@ -17733,7 +17818,7 @@ using Catch::Detail::Approx;
 #define CATCH_FAIL_CHECK( ... ) (void)(0)
 #define CATCH_SUCCEED( ... ) (void)(0)
 
-#define CATCH_ANON_TEST_CASE() INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ))
+#define CATCH_ANON_TEST_CASE() INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ ))
 
 #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR
 #define CATCH_TEMPLATE_TEST_CASE( ... ) INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION(__VA_ARGS__)
@@ -17756,8 +17841,8 @@ using Catch::Detail::Approx;
 #endif
 
 // "BDD-style" convenience wrappers
-#define CATCH_SCENARIO( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ))
-#define CATCH_SCENARIO_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_METHOD_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ), className )
+#define CATCH_SCENARIO( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ ))
+#define CATCH_SCENARIO_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_METHOD_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ ), className )
 #define CATCH_GIVEN( desc )
 #define CATCH_AND_GIVEN( desc )
 #define CATCH_WHEN( desc )
@@ -17807,8 +17892,8 @@ using Catch::Detail::Approx;
 #define WARN( msg ) (void)(0)
 #define CAPTURE( msg ) (void)(0)
 
-#define TEST_CASE( ... )  INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ))
-#define TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ))
+#define TEST_CASE( ... )  INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ ))
+#define TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ ))
 #define METHOD_AS_TEST_CASE( method, ... )
 #define REGISTER_TEST_CASE( Function, ... ) (void)(0)
 #define SECTION( ... )
@@ -17816,7 +17901,7 @@ using Catch::Detail::Approx;
 #define FAIL( ... ) (void)(0)
 #define FAIL_CHECK( ... ) (void)(0)
 #define SUCCEED( ... ) (void)(0)
-#define ANON_TEST_CASE() INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ))
+#define ANON_TEST_CASE() INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ ))
 
 #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR
 #define TEMPLATE_TEST_CASE( ... ) INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION(__VA_ARGS__)
@@ -17846,8 +17931,8 @@ using Catch::Detail::Approx;
 #define CATCH_TRANSLATE_EXCEPTION( signature ) INTERNAL_CATCH_TRANSLATE_EXCEPTION_NO_REG( INTERNAL_CATCH_UNIQUE_NAME( catch_internal_ExceptionTranslator ), signature )
 
 // "BDD-style" convenience wrappers
-#define SCENARIO( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ) )
-#define SCENARIO_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_METHOD_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ), className )
+#define SCENARIO( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ ) )
+#define SCENARIO_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_METHOD_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ ), className )
 
 #define GIVEN( desc )
 #define AND_GIVEN( desc )
@@ -17878,4 +17963,3 @@ using Catch::Detail::Approx;
 // end catch_reenable_warnings.h
 // end catch.hpp
 #endif // TWOBLUECUBES_SINGLE_INCLUDE_CATCH_HPP_INCLUDED
-

tests/cne_test.cpp

@@ -94,7 +94,7 @@ TEST_CASE("CNELevyFunctionN13Test", "[CNETest]")
 TEST_CASE("CNEHimmelblauFunctionTest", "[CNETest]")
 {
   HimmelblauFunction f;
-  CNE optimizer(450, 1500, 0.3, 0.3, 0.3, 1e-7);
+  CNE optimizer(650, 3000, 0.3, 0.3, 0.3, 1e-7);
 
   arma::mat coordinates = arma::mat("2; 1");
   optimizer.Optimize(f, coordinates);

tests/demon_adam_test.cpp

@@ -0,0 +1,75 @@
+/**
+ * @file demon_adam_test.cpp
+ * @author Marcus Edel
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#include <ensmallen.hpp>
+#include "catch.hpp"
+#include "test_function_tools.hpp"
+
+using namespace ens;
+using namespace ens::test;
+
+/**
+ * Run DemonAdam on logistic regression and make sure the results are
+ * acceptable.
+ */
+TEST_CASE("DemonAdamLogisticRegressionTest", "[DemonAdamTest]")
+{
+  DemonAdam optimizer(0.2, 32, 0.9, 0.9, 0.999, 1e-8,
+      10000, 1e-9, true, true, true);
+  LogisticRegressionFunctionTest(optimizer, 0.003, 0.006, 6);
+}
+
+/**
+ * Test the Adam optimizer on the Sphere function.
+ */
+TEST_CASE("DemonAdamSphereFunctionTest", "[DemonAdamTest]")
+{
+  SphereFunction f(2);
+  DemonAdam optimizer(0.5, 2, 0.9);
+  FunctionTest<SphereFunction, arma::mat>(optimizer, 1.0, 0.1);
+}
+
+/**
+ * Test the DemonAdam optimizer on the Matyas function.
+ */
+TEST_CASE("DemonAdamMatyasFunctionTest", "[DemonAdamTest]")
+{
+  DemonAdam optimizer(0.5, 1, 0.9);
+  FunctionTest<MatyasFunction, arma::mat>(optimizer, 0.1, 0.01);
+}
+
+/**
+ * Test the Adam optimizer on the Sphere function.
+ */
+TEST_CASE("DemonAdamSphereFunctionTestFloat", "[DemonAdamTest]")
+{
+  DemonAdam optimizer(0.5, 2, 0.9);
+  FunctionTest<SphereFunction, arma::sp_mat>(optimizer, 1.0, 0.1);
+}
+
+/**
+ * Test the DemonAdam optimizer on the Matyas function.
+ */
+TEST_CASE("DemonAdamMatyasFunctionTestFloat", "[DemonAdamTest]")
+{
+  DemonAdam optimizer(0.5, 1, 0.9);
+  FunctionTest<MatyasFunction, arma::fmat>(optimizer, 0.1, 0.01);
+}
+
+/**
+ * Run DemonAdam (AdaMax update) on logistic regression and make sure the
+ * results are acceptable.
+ */
+TEST_CASE("DemonAdaMaxLogisticRegressionTest", "[DemonAdamTest]")
+{
+  DemonAdamType<AdaMaxUpdate> optimizer(0.5, 10, 0.9, 0.9, 0.999, 1e-8,
+      10000, 1e-9, true, true, true);
+  LogisticRegressionFunctionTest(optimizer, 0.003, 0.006, 3);
+}

tests/demon_sgd_test.cpp

@@ -0,0 +1,43 @@
+/**
+ * @file demon_sgd_test.cpp
+ * @author Marcus Edel
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#include <ensmallen.hpp>
+#include "catch.hpp"
+#include "test_function_tools.hpp"
+
+using namespace ens;
+using namespace ens::test;
+
+/**
+ * Run DemonSGD on logistic regression and make sure the results are
+ * acceptable.
+ */
+TEST_CASE("DemonSGDLogisticRegressionTest", "[DemonSGDTest]")
+{
+  DemonSGD optimizer(0.1, 32, 0.9, 1000000, 1e-9, true, true, true);
+  LogisticRegressionFunctionTest(optimizer, 0.003, 0.006, 6);
+}
+
+/**
+ * Tests the DemonSGD optimizer using a simple test function.
+ */
+TEST_CASE("DemonSGDSimpleTestFunctionFloat", "[DemonSGDTest]")
+{
+  SGDTestFunction f;
+  DemonSGD optimizer(1e-2, 1, 0.9, 400000);
+
+  arma::fmat coordinates = f.GetInitialPoint<arma::fmat>();
+  optimizer.Optimize(f, coordinates);
+
+  REQUIRE(coordinates(0) == Approx(0.0).margin(0.1));
+  REQUIRE(coordinates(1) == Approx(0.0).margin(0.1));
+  REQUIRE(coordinates(2) == Approx(0.0).margin(0.1));
+}
+

tests/indicators_test.cpp

@@ -0,0 +1,97 @@
+/**
+ * @file indicators_test.cpp
+ * @author Nanubala Gnana Sai
+ *
+ * Test file for all the indicators: Epsilon, IGD+.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#include <ensmallen.hpp>
+#include "catch.hpp"
+
+using namespace ens;
+using namespace ens::test;
+
+/**
+ * Calculates the Epsilon metric for the pair of fronts.
+ * Tests for data of type double.
+ * The reference numerical results have been taken from hand calculated values.
+ * Refer the IPynb notebook in https://github.com/mlpack/ensmallen/pull/285
+ * for more.
+ */
+TEST_CASE("EpsilonDoubleTest", "[IndicatorsTest]")
+{
+  arma::cube referenceFront(2, 1, 3);
+  double tol = 1e-10;
+  referenceFront.slice(0) = arma::vec{0.01010101, 0.89949622};
+  referenceFront.slice(1) = arma::vec{0.02020202, 0.85786619};
+  referenceFront.slice(2) = arma::vec{0.03030303, 0.82592234};
+  arma::cube front = referenceFront * 1.1;
+  double eps =  Epsilon::Evaluate(front, referenceFront);
+
+  REQUIRE(eps == Approx(1.1).margin(tol));
+}
+
+/**
+ * Calculates the Epsilon metric for the pair of fronts.
+ * Tests for data of type float.
+ * The reference numerical results have been taken from hand calculated values.
+ * Refer the IPynb notebook in https://github.com/mlpack/ensmallen/pull/285
+ * for more.
+ */
+TEST_CASE("EpsilonFloatTest", "[IndicatorsTest]")
+{
+  arma::fcube referenceFront(2, 1, 3);
+  float tol = 1e-10;
+  referenceFront.slice(0) = arma::fvec{0.01010101f, 0.89949622f};
+  referenceFront.slice(1) = arma::fvec{0.02020202f, 0.85786619f};
+  referenceFront.slice(2) = arma::fvec{0.03030303f, 0.82592234f};
+  arma::fcube front = referenceFront * 1.1;
+  double eps =  Epsilon::Evaluate(front, referenceFront);
+
+  REQUIRE(eps == Approx(1.1).margin(tol));
+}
+
+/**
+ * Calculates the IGD+ metric for the pair of fronts.
+ * Tests for data of type double.
+ * The reference numerical results have been taken from hand calculated values.
+ * Refer the IPynb notebook in https://github.com/mlpack/ensmallen/pull/285
+ * for more.
+ */
+TEST_CASE("IGDPlusDoubleTest", "[IndicatorsTest]")
+{
+  arma::cube referenceFront(2, 1, 3);
+  double tol = 1e-10;
+  referenceFront.slice(0) = arma::vec{0.01010101, 0.89949622};
+  referenceFront.slice(1) = arma::vec{0.02020202, 0.85786619};
+  referenceFront.slice(2) = arma::vec{0.03030303, 0.82592234};
+  arma::cube front = referenceFront * 1.1;
+  double igdPlus = IGDPlus::Evaluate(front, referenceFront);
+
+  REQUIRE(igdPlus == Approx(0.05329735411078149).margin(tol));
+}
+
+/**
+ * Calculates the IGD+ metric for the pair of fronts.
+ * Tests for data of type float.
+ * The reference numerical results have been taken from hand calculated values.
+ * Refer the IPynb notebook in https://github.com/mlpack/ensmallen/pull/285
+ * for more.
+ */
+TEST_CASE("IGDPlusFloatTest", "[IndicatorsTest]")
+{
+  arma::fcube referenceFront(2, 1, 3);
+  float tol = 1e-10;
+  referenceFront.slice(0) = arma::fvec{0.01010101f, 0.89949622f};
+  referenceFront.slice(1) = arma::fvec{0.02020202f, 0.85786619f};
+  referenceFront.slice(2) = arma::fvec{0.03030303f, 0.82592234f};
+  arma::fcube front = referenceFront * 1.1;
+  float igdPlus = IGDPlus::Evaluate(front, referenceFront);
+
+  REQUIRE(igdPlus == Approx(0.05329735411078149).margin(tol));
+}

tests/main.cpp

@@ -17,18 +17,23 @@
 
 int main(int argc, char** argv)
 {
-  /**
-   * Uncomment these three lines if you want to test with different random seeds
-   * each run.  This is good for ensuring that a test's tolerance is sufficient
-   * across many different runs.
-   */
-  //size_t seed = std::time(NULL);
-  //srand((unsigned int) seed);
-  //arma::arma_rng::set_seed(seed);
+  Catch::Session session;
+  const int returnCode = session.applyCommandLine(argc, argv);
+  // Check for a command line error.
+  if (returnCode != 0)
+    return returnCode;
 
   std::cout << "ensmallen version: " << ens::version::as_string() << std::endl;
-
   std::cout << "armadillo version: " << arma::arma_version::as_string() << std::endl;
 
-  return Catch::Session().run(argc, argv);
+  // Use Catch2 command-line to set the random seed.
+  // -rng-seed <'time'|number>
+  // If a number is provided this is used directly as the seed. Alternatively
+  // if the keyword 'time' is provided then the result of calling std::time(0)
+  // is used.
+  const size_t seed = session.config().rngSeed();
+  srand((unsigned int) seed);
+  arma::arma_rng::set_seed(seed);
+
+  return session.run();
 }

tests/moead_test.cpp

@@ -0,0 +1,630 @@
+/**
+ * @file moead_test.cpp
+ * @author Nanubala Gnana Sai
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#include <ensmallen.hpp>
+#include "catch.hpp"
+#include "test_function_tools.hpp"
+
+using namespace ens;
+using namespace ens::test;
+using namespace std;
+
+/**
+ * Checks if low <= value <= high. Used by MOEADFonsecaFlemingTest.
+ *
+ * @param value The value being checked.
+ * @param low The lower bound.
+ * @param high The upper bound.
+ * @param roundoff To round off precision.
+ * @tparam The type of elements in the population set.
+ * @return true if value lies in the range [low, high].
+ * @return false if value does not lie in the range [low, high].
+ */
+template<typename ElemType>
+bool IsInBounds(const ElemType& value,
+                const ElemType& low,
+                const ElemType& high,
+                const ElemType& roundoff)
+{
+  return !(value < (low - roundoff)) && !((high + roundoff) < value);
+}
+
+/**
+ * Optimize for the Schaffer N.1 function using NSGA-II optimizer.
+ * Tests for data of type double.
+ */
+TEST_CASE("MOEADSchafferN1DoubleTest", "[MOEADTest]")
+{
+  SchafferFunctionN1<arma::mat> SCH;
+  const double lowerBound = -1000;
+  const double upperBound = 1000;
+  const double expectedLowerBound = 0.0;
+  const double expectedUpperBound = 2.0;
+
+  DefaultMOEAD opt(
+        300, // Population size.
+        300,  // Max generations.
+        1.0,  // Crossover probability.
+        0.9, // Probability of sampling from neighbor.
+        20, // Neighborhood size.
+        20, // Perturbation index.
+        0.5, // Differential weight.
+        2, // Max childrens to replace parents.
+        1E-10, // epsilon.
+        lowerBound, // Lower bound.
+        upperBound // Upper bound.
+      );
+
+  typedef decltype(SCH.objectiveA) ObjectiveTypeA;
+  typedef decltype(SCH.objectiveB) ObjectiveTypeB;
+
+  // We allow a few trials in case of poor convergence.
+  bool success = false;
+  for (size_t trial = 0; trial < 5; ++trial)
+  {
+    arma::mat coords = SCH.GetInitialPoint();
+    std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = SCH.GetObjectives();
+
+    opt.Optimize(objectives, coords);
+    arma::cube paretoSet= opt.ParetoSet();
+
+    bool allInRange = true;
+
+    for (size_t solutionIdx = 0; solutionIdx < paretoSet.n_slices; ++solutionIdx)
+    {
+      double val = arma::as_scalar(paretoSet.slice(solutionIdx));
+      if (!IsInBounds<double>(val, expectedLowerBound, expectedUpperBound, 0.1))
+      {
+        allInRange = false;
+        break;
+      }
+    }
+
+    if (allInRange)
+    {
+      success = true;
+      break;
+    }
+  }
+
+  REQUIRE(success == true);
+}
+
+/**
+ * Optimize for the Schaffer N.1 function using NSGA-II optimizer.
+ * Tests for data of type double.
+ */
+TEST_CASE("MOEADSchafferN1TestVectorDoubleBounds", "[MOEADTest]")
+{
+  // This test can be a little flaky, so we try it a few times.
+  SchafferFunctionN1<arma::mat> SCH;
+  const arma::vec lowerBound = {-1000};
+  const arma::vec upperBound = {1000};
+  const double expectedLowerBound = 0.0;
+  const double expectedUpperBound = 2.0;
+
+  DefaultMOEAD opt(
+        300, // Population size.
+        300,  // Max generations.
+        1.0,  // Crossover probability.
+        0.9, // Probability of sampling from neighbor.
+        20, // Neighborhood size.
+        20, // Perturbation index.
+        0.5, // Differential weight.
+        2, // Max childrens to replace parents.
+        1E-10, // epsilon.
+        lowerBound, // Lower bound.
+        upperBound // Upper bound.
+      );
+
+  typedef decltype(SCH.objectiveA) ObjectiveTypeA;
+  typedef decltype(SCH.objectiveB) ObjectiveTypeB;
+
+  bool success = false;
+  for (size_t trial = 0; trial < 3; ++trial)
+  {
+    arma::mat coords = SCH.GetInitialPoint();
+    std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = SCH.GetObjectives();
+
+    opt.Optimize(objectives, coords);
+    arma::cube paretoSet = opt.ParetoSet();
+
+    bool allInRange = true;
+
+    for (size_t solutionIdx = 0; solutionIdx < paretoSet.n_slices; ++solutionIdx)
+    {
+      double val = arma::as_scalar(paretoSet.slice(solutionIdx));
+      if (!IsInBounds<double>(val, expectedLowerBound, expectedUpperBound, 0.1))
+      {
+        allInRange = false;
+        break;
+      }
+    }
+
+    if (allInRange)
+    {
+      success = true;
+      break;
+    }
+  }
+
+  REQUIRE(success == true);
+}
+
+/**
+ * Optimize for the Fonseca Fleming function using NSGA-II optimizer.
+ * Tests for data of type double.
+ */
+TEST_CASE("MOEADFonsecaFlemingDoubleTest", "[MOEADTest]")
+{
+  FonsecaFlemingFunction<arma::mat> FON;
+  const double lowerBound = -4;
+  const double upperBound = 4;
+  const double expectedLowerBound = -1.0 / sqrt(3);
+  const double expectedUpperBound = 1.0 / sqrt(3);
+
+  DefaultMOEAD opt(
+        300,  // Max generations.
+        300,  // Max generations.
+        1.0,  // Crossover probability.
+        0.9, // Probability of sampling from neighbor.
+        20, // Neighborhood size.
+        20, // Perturbation index.
+        0.5, // Differential weight.
+        2, // Max childrens to replace parents.
+        1E-10, // epsilon.
+        lowerBound, // Lower bound.
+        upperBound // Upper bound.
+      );
+  typedef decltype(FON.objectiveA) ObjectiveTypeA;
+  typedef decltype(FON.objectiveB) ObjectiveTypeB;
+
+  arma::mat coords = FON.GetInitialPoint();
+  std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = FON.GetObjectives();
+
+  opt.Optimize(objectives, coords);
+  arma::cube paretoSet = opt.ParetoSet();
+
+  bool allInRange = true;
+
+  for (size_t solutionIdx = 0; solutionIdx < paretoSet.n_slices; ++solutionIdx)
+  {
+    const arma::mat solution = paretoSet.slice(solutionIdx);
+    double valX = arma::as_scalar(solution(0));
+    double valY = arma::as_scalar(solution(1));
+    double valZ = arma::as_scalar(solution(2));
+
+    if (!IsInBounds<double>(valX, expectedLowerBound, expectedUpperBound, 0.1) ||
+        !IsInBounds<double>(valY, expectedLowerBound, expectedUpperBound, 0.1) ||
+        !IsInBounds<double>(valZ, expectedLowerBound, expectedUpperBound, 0.1))
+    {
+      allInRange = false;
+      break;
+    }
+  }
+
+  REQUIRE(allInRange);
+}
+
+/**
+ * Optimize for the Fonseca Fleming function using NSGA-II optimizer.
+ * Tests for data of type double.
+ */
+TEST_CASE("MOEADFonsecaFlemingTestVectorDoubleBounds", "[MOEADTest]")
+{
+  FonsecaFlemingFunction<arma::mat> FON;
+  const arma::vec lowerBound = {-4, -4, -4};
+  const arma::vec upperBound = {4, 4, 4};
+  const double expectedLowerBound = -1.0 / sqrt(3);
+  const double expectedUpperBound = 1.0 / sqrt(3);
+
+  DefaultMOEAD opt(
+        300,  // Max generations.
+        300,  // Max generations.
+        1.0,  // Crossover probability.
+        0.9, // Probability of sampling from neighbor.
+        20, // Neighborhood size.
+        20, // Perturbation index.
+        0.5, // Differential weight.
+        2, // Max childrens to replace parents.
+        1E-10, // epsilon.
+        lowerBound, // Lower bound.
+        upperBound // Upper bound.
+      );
+  typedef decltype(FON.objectiveA) ObjectiveTypeA;
+  typedef decltype(FON.objectiveB) ObjectiveTypeB;
+
+  arma::mat coords = FON.GetInitialPoint();
+  std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = FON.GetObjectives();
+
+  opt.Optimize(objectives, coords);
+  arma::cube paretoSet = opt.ParetoSet();
+
+  bool allInRange = true;
+
+  for (size_t solutionIdx = 0; solutionIdx < paretoSet.n_slices; ++solutionIdx)
+  {
+    const arma::mat solution = paretoSet.slice(solutionIdx);
+    double valX = arma::as_scalar(solution(0));
+    double valY = arma::as_scalar(solution(1));
+    double valZ = arma::as_scalar(solution(2));
+
+    if (!IsInBounds<double>(valX, expectedLowerBound, expectedUpperBound, 0.1) ||
+        !IsInBounds<double>(valY, expectedLowerBound, expectedUpperBound, 0.1) ||
+        !IsInBounds<double>(valZ, expectedLowerBound, expectedUpperBound, 0.1))
+    {
+      allInRange = false;
+      break;
+    }
+  }
+
+  REQUIRE(allInRange);
+}
+
+/**
+ * Optimize for the Schaffer N.1 function using NSGA-II optimizer.
+ * Tests for data of type float.
+ */
+TEST_CASE("MOEADSchafferN1FloatTest", "[MOEADTest]")
+{
+  SchafferFunctionN1<arma::fmat> SCH;
+  const double lowerBound = -1000;
+  const double upperBound = 1000;
+  const double expectedLowerBound = 0.0;
+  const double expectedUpperBound = 2.0;
+
+  DefaultMOEAD opt(
+        300, // Population size.
+        300,  // Max generations.
+        1.0,  // Crossover probability.
+        0.9, // Probability of sampling from neighbor.
+        20, // Neighborhood size.
+        20, // Perturbation index.
+        0.5, // Differential weight.
+        2, // Max childrens to replace parents.
+        1E-10, // epsilon.
+        lowerBound, // Lower bound.
+        upperBound // Upper bound.
+      );
+
+  typedef decltype(SCH.objectiveA) ObjectiveTypeA;
+  typedef decltype(SCH.objectiveB) ObjectiveTypeB;
+
+  // We allow a few trials in case of poor convergence.
+  bool success = false;
+  for (size_t trial = 0; trial < 3; ++trial)
+  {
+    arma::fmat coords = SCH.GetInitialPoint();
+    std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = SCH.GetObjectives();
+
+    opt.Optimize(objectives, coords);
+    arma::fcube paretoSet = arma::conv_to<arma::fcube>::from(opt.ParetoSet());
+
+    bool allInRange = true;
+
+    for (size_t solutionIdx = 0; solutionIdx < paretoSet.n_slices; ++solutionIdx)
+    {
+      float val = arma::as_scalar(paretoSet.slice(solutionIdx));
+      if (!IsInBounds<float>(val, expectedLowerBound, expectedUpperBound, 0.1))
+      {
+        allInRange = false;
+        break;
+      }
+    }
+
+    if (allInRange)
+    {
+      success = true;
+      break;
+    }
+  }
+
+  REQUIRE(success == true);
+}
+
+/**
+ * Optimize for the Schaffer N.1 function using NSGA-II optimizer.
+ * Tests for data of type float.
+ */
+TEST_CASE("MOEADSchafferN1TestVectorFloatBounds", "[MOEADTest]")
+{
+  // This test can be a little flaky, so we try it a few times.
+  SchafferFunctionN1<arma::fmat> SCH;
+  const arma::vec lowerBound = {-1000};
+  const arma::vec upperBound = {1000};
+  const double expectedLowerBound = 0.0;
+  const double expectedUpperBound = 2.0;
+
+  DefaultMOEAD opt(
+        300, // Population size.
+        300,  // Max generations.
+        1.0,  // Crossover probability.
+        0.9, // Probability of sampling from neighbor.
+        20, // Neighborhood size.
+        20, // Perturbation index.
+        0.5, // Differential weight.
+        2, // Max childrens to replace parents.
+        1E-10, // epsilon.
+        lowerBound, // Lower bound.
+        upperBound // Upper bound.
+      );
+
+  typedef decltype(SCH.objectiveA) ObjectiveTypeA;
+  typedef decltype(SCH.objectiveB) ObjectiveTypeB;
+
+  bool success = false;
+  for (size_t trial = 0; trial < 3; ++trial)
+  {
+    arma::fmat coords = SCH.GetInitialPoint();
+    std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = SCH.GetObjectives();
+
+    opt.Optimize(objectives, coords);
+    arma::fcube paretoSet = arma::conv_to<arma::fcube>::from(opt.ParetoSet());
+
+    bool allInRange = true;
+
+    for (size_t solutionIdx = 0; solutionIdx < paretoSet.n_slices; ++solutionIdx)
+    {
+      float val = arma::as_scalar(paretoSet.slice(solutionIdx));
+      if (!IsInBounds<float>(val, expectedLowerBound, expectedUpperBound, 0.1))
+      {
+        allInRange = false;
+        break;
+      }
+    }
+
+    if (allInRange)
+    {
+      success = true;
+      break;
+    }
+  }
+
+  REQUIRE(success == true);
+}
+
+/**
+ * Optimize for the Fonseca Fleming function using NSGA-II optimizer.
+ * Tests for data of type float.
+ */
+TEST_CASE("MOEADFonsecaFlemingFloatTest", "[MOEADTest]")
+{
+  FonsecaFlemingFunction<arma::fmat> FON;
+  const double lowerBound = -4;
+  const double upperBound = 4;
+  const float expectedLowerBound = -1.0 / sqrt(3);
+  const float expectedUpperBound = 1.0 / sqrt(3);
+
+  DefaultMOEAD opt(
+        300,  // Max generations.
+        300,  // Max generations.
+        1.0,  // Crossover probability.
+        0.9, // Probability of sampling from neighbor.
+        20, // Neighborhood size.
+        20, // Perturbation index.
+        0.5, // Differential weight.
+        2, // Max childrens to replace parents.
+        1E-10, // epsilon.
+        lowerBound, // Lower bound.
+        upperBound // Upper bound.
+      );
+  typedef decltype(FON.objectiveA) ObjectiveTypeA;
+  typedef decltype(FON.objectiveB) ObjectiveTypeB;
+
+  arma::fmat coords = FON.GetInitialPoint();
+  std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = FON.GetObjectives();
+
+  opt.Optimize(objectives, coords);
+  arma::fcube paretoSet = arma::conv_to<arma::fcube>::from(opt.ParetoSet());
+
+  bool allInRange = true;
+
+  for (size_t solutionIdx = 0; solutionIdx < paretoSet.n_slices; ++solutionIdx)
+  {
+    const arma::fmat solution = paretoSet.slice(solutionIdx);
+    float valX = arma::as_scalar(solution(0));
+    float valY = arma::as_scalar(solution(1));
+    float valZ = arma::as_scalar(solution(2));
+
+    if (!IsInBounds<float>(valX, expectedLowerBound, expectedUpperBound, 0.1) ||
+        !IsInBounds<float>(valY, expectedLowerBound, expectedUpperBound, 0.1) ||
+        !IsInBounds<float>(valZ, expectedLowerBound, expectedUpperBound, 0.1))
+    {
+      allInRange = false;
+      break;
+    }
+  }
+
+  REQUIRE(allInRange);
+}
+
+/**
+ * Optimize for the Fonseca Fleming function using NSGA-II optimizer.
+ * Tests for data of type float.
+ */
+TEST_CASE("MOEADFonsecaFlemingTestVectorFloatBounds", "[MOEADTest]")
+{
+  FonsecaFlemingFunction<arma::fmat> FON;
+  const arma::vec lowerBound = {-4, -4, -4};
+  const arma::vec upperBound = {4, 4, 4};
+  const float expectedLowerBound = -1.0 / sqrt(3);
+  const float expectedUpperBound = 1.0 / sqrt(3);
+
+  DefaultMOEAD opt(
+        300,  // Max generations.
+        300,  // Max generations.
+        1.0,  // Crossover probability.
+        0.9, // Probability of sampling from neighbor.
+        20, // Neighborhood size.
+        20, // Perturbation index.
+        0.5, // Differential weight.
+        2, // Max childrens to replace parents.
+        1E-10, // epsilon.
+        lowerBound, // Lower bound.
+        upperBound // Upper bound.
+      );
+  typedef decltype(FON.objectiveA) ObjectiveTypeA;
+  typedef decltype(FON.objectiveB) ObjectiveTypeB;
+
+  arma::fmat coords = FON.GetInitialPoint();
+  std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = FON.GetObjectives();
+
+  opt.Optimize(objectives, coords);
+  arma::fcube paretoSet = arma::conv_to<arma::fcube>::from(opt.ParetoSet());
+
+  bool allInRange = true;
+
+  for (size_t solutionIdx = 0; solutionIdx < paretoSet.n_slices; ++solutionIdx)
+  {
+    const arma::fmat solution = paretoSet.slice(solutionIdx);
+    float valX = arma::as_scalar(solution(0));
+    float valY = arma::as_scalar(solution(1));
+    float valZ = arma::as_scalar(solution(2));
+
+    if (!IsInBounds<float>(valX, expectedLowerBound, expectedUpperBound, 0.1) ||
+        !IsInBounds<float>(valY, expectedLowerBound, expectedUpperBound, 0.1) ||
+        !IsInBounds<float>(valZ, expectedLowerBound, expectedUpperBound, 0.1))
+    {
+      allInRange = false;
+      break;
+    }
+  }
+
+  REQUIRE(allInRange);
+}
+
+/**
+ * Test against the first problem of ZDT Test Suite.  ZDT-1 is a 30 
+ * variable-2 objective problem with a convex Pareto Front. 
+ *
+ * NOTE: For the sake of runtime, only ZDT-1 is tested against the
+ * algorithm. Others have been tested separately.
+ *
+ * We run the test multiple times, since it sometimes fails, in order to get the
+ * probability of failure down.
+ */
+TEST_CASE("MOEADZDTONETest", "[MOEADTest]")
+{
+  //! Parameters taken from original ZDT Paper.
+  ZDT1<> ZDT_ONE(100);
+  const double lowerBound = 0;
+  const double upperBound = 1;
+
+  DefaultMOEAD opt(
+      300, // Population size.
+      150,  // Max generations.
+      1.0,  // Crossover probability.
+      0.9, // Probability of sampling from neighbor.
+      20, // Neighborhood size.
+      20, // Perturbation index.
+      0.5, // Differential weight.
+      2, // Max childrens to replace parents.
+      1E-10, // epsilon.
+      lowerBound, // Lower bound.
+      upperBound // Upper bound.
+    );
+
+  typedef decltype(ZDT_ONE.objectiveF1) ObjectiveTypeA;
+  typedef decltype(ZDT_ONE.objectiveF2) ObjectiveTypeB;
+
+  const size_t trials = 8;
+  for (size_t trial = 0; trial < trials; ++trial)
+  {
+    arma::mat coords = ZDT_ONE.GetInitialPoint();
+    std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives =
+        ZDT_ONE.GetObjectives();
+
+    opt.Optimize(objectives, coords);
+
+    //! Refer the ZDT_ONE implementation for g objective implementation.
+    //! The optimal g value is taken from the docs of ZDT_ONE.
+    size_t numVariables = coords.size();
+    double sum = arma::accu(coords(arma::span(1, numVariables - 1), 0));
+    const double g = 1.0 + 9.0 * sum / (static_cast<double>(numVariables - 1));
+    if (trial < trials - 1 && g != Approx(1.0).margin(0.99))
+      continue;
+
+    REQUIRE(g == Approx(1.0).margin(0.99));
+    break;
+  }
+}
+
+/**
+ * Check if the final population lies in the optimal region in variable space.
+ *
+ * @param paretoSet The final population in variable space.
+ */
+bool VariableBoundsCheck(const arma::cube& paretoSet)
+{
+  bool inBounds = true;
+  const arma::mat regions{
+    {0.0, 0.182228780, 0.4093136748,
+      0.6183967944, 0.8233317983},
+    {0.0830015349, 0.2577623634, 0.4538821041,
+      0.6525117038, 0.8518328654}
+  };
+
+  for (size_t pointIdx = 0; pointIdx < paretoSet.n_slices; ++pointIdx)
+  {
+    const arma::mat& point = paretoSet.slice(pointIdx);
+    const double firstVariable = point(0, 0);
+
+    const bool notInRegion0 = !IsInBounds<double>(firstVariable, regions(0, 0), regions(1, 0), 1e-2);
+    const bool notInRegion1 = !IsInBounds<double>(firstVariable, regions(0, 1), regions(1, 1), 1e-2);
+    const bool notInRegion2 = !IsInBounds<double>(firstVariable, regions(0, 2), regions(1, 2), 1e-2);
+    const bool notInRegion3 = !IsInBounds<double>(firstVariable, regions(0, 3), regions(1, 3), 1e-2);
+    const bool notInRegion4 = !IsInBounds<double>(firstVariable, regions(0, 4), regions(1, 4), 1e-2);
+
+    if (notInRegion0 && notInRegion1 && notInRegion2 && notInRegion3 && notInRegion4)
+    {
+      inBounds = false;
+      break;
+    }
+  }
+
+  return inBounds;
+}
+
+/**
+ * Test DirichletMOEAD against the third problem of ZDT Test Suite. ZDT-3 is a 30 
+ * variable-2 objective problem with disconnected Pareto Fronts. 
+ */
+TEST_CASE("MOEADDIRICHLETZDT3Test", "[MOEADTest]")
+{
+  //! Parameters taken from original ZDT Paper.
+  ZDT3<> ZDT_THREE(300);
+  const double lowerBound = 0;
+  const double upperBound = 1;
+
+  DirichletMOEAD opt(
+      300, // Population size.
+      300,  // Max generations.
+      1.0,  // Crossover probability.
+      0.9, // Probability of sampling from neighbor.
+      20, // Neighborhood size.
+      20, // Perturbation index.
+      0.5, // Differential weight.
+      2, // Max childrens to replace parents.
+      1E-10, // epsilon.
+      lowerBound, // Lower bound.
+      upperBound // Upper bound.
+    );
+
+  typedef decltype(ZDT_THREE.objectiveF1) ObjectiveTypeA;
+  typedef decltype(ZDT_THREE.objectiveF2) ObjectiveTypeB;
+
+  arma::mat coords = ZDT_THREE.GetInitialPoint();
+  std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = ZDT_THREE.GetObjectives();
+
+  opt.Optimize(objectives, coords);
+
+  const arma::cube& finalPopulation = opt.ParetoSet();
+  REQUIRE(VariableBoundsCheck(finalPopulation));
+}

tests/nsga2_test.cpp

@@ -1,6 +1,7 @@
 /**
  * @file nsga2_test.cpp
  * @author Sayan Goswami
+ * @author Nanubala Gnana Sai
  *
  * ensmallen is free software; you may redistribute it and/or modify it under
  * the terms of the 3-clause BSD license.  You should have received a copy of
@@ -22,24 +23,30 @@ using namespace std;
  * @param value The value being checked.
  * @param low The lower bound.
  * @param high The upper bound.
+ * @tparam The type of elements in the population set.
  * @return true if value lies in the range [low, high].
  * @return false if value does not lie in the range [low, high].
  */
-bool IsInBounds(const double& value, const double& low, const double& high)
+template<typename ElemType>
+bool IsInBounds(const ElemType& value, const ElemType& low, const ElemType& high)
 {
-  return !(value < low) && !(high < value);
+  ElemType roundoff = 0.1;
+  return !(value < (low - roundoff)) && !((high + roundoff) < value);
 }
 
 /**
  * Optimize for the Schaffer N.1 function using NSGA-II optimizer.
+ * Tests for data of type double.
  */
-TEST_CASE("NSGA2SchafferN1Test", "[NSGA2Test]")
+TEST_CASE("NSGA2SchafferN1DoubleTest", "[NSGA2Test]")
 {
   SchafferFunctionN1<arma::mat> SCH;
   const double lowerBound = -1000;
   const double upperBound = 1000;
+  const double expectedLowerBound = 0.0;
+  const double expectedUpperBound = 2.0;
 
-  NSGA2 opt(20, 5000, 0.5, 0.5, 1e-3, 1e-6, lowerBound, upperBound);
+  NSGA2 opt(20, 300, 0.5, 0.5, 1e-3, 1e-6, lowerBound, upperBound);
 
   typedef decltype(SCH.objectiveA) ObjectiveTypeA;
   typedef decltype(SCH.objectiveB) ObjectiveTypeB;
@@ -52,15 +59,14 @@ TEST_CASE("NSGA2SchafferN1Test", "[NSGA2Test]")
     std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = SCH.GetObjectives();
 
     opt.Optimize(objectives, coords);
-    std::vector<arma::mat> bestFront = opt.Front();
+    arma::cube paretoSet = opt.ParetoSet();
 
     bool allInRange = true;
 
-    for (arma::mat solution: bestFront)
+    for (size_t solutionIdx = 0; solutionIdx < paretoSet.n_slices; ++solutionIdx)
     {
-      double val = arma::as_scalar(solution);
-
-      if (val < 0.0 || val > 2.0)
+      double val = arma::as_scalar(paretoSet.slice(solutionIdx));
+      if (!IsInBounds<double>(val, expectedLowerBound, expectedUpperBound))
       {
         allInRange = false;
         break;
@@ -79,15 +85,18 @@ TEST_CASE("NSGA2SchafferN1Test", "[NSGA2Test]")
 
 /**
  * Optimize for the Schaffer N.1 function using NSGA-II optimizer.
+ * Tests for data of type double.
  */
-TEST_CASE("NSGA2SchafferN1TestVectorBounds", "[NSGA2Test]")
+TEST_CASE("NSGA2SchafferN1TestVectorDoubleBounds", "[NSGA2Test]")
 {
   // This test can be a little flaky, so we try it a few times.
   SchafferFunctionN1<arma::mat> SCH;
   const arma::vec lowerBound = {-1000};
   const arma::vec upperBound = {1000};
+  const double expectedLowerBound = 0.0;
+  const double expectedUpperBound = 2.0;
 
-  NSGA2 opt(20, 5000, 0.5, 0.5, 1e-3, 1e-6, lowerBound, upperBound);
+  NSGA2 opt(20, 300, 0.5, 0.5, 1e-3, 1e-6, lowerBound, upperBound);
 
   typedef decltype(SCH.objectiveA) ObjectiveTypeA;
   typedef decltype(SCH.objectiveB) ObjectiveTypeB;
@@ -99,15 +108,14 @@ TEST_CASE("NSGA2SchafferN1TestVectorBounds", "[NSGA2Test]")
     std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = SCH.GetObjectives();
 
     opt.Optimize(objectives, coords);
-    std::vector<arma::mat> bestFront = opt.Front();
+    arma::cube paretoSet = opt.ParetoSet();
 
     bool allInRange = true;
 
-    for (arma::mat solution: bestFront)
+    for (size_t solutionIdx = 0; solutionIdx < paretoSet.n_slices; ++solutionIdx)
     {
-      double val = arma::as_scalar(solution);
-
-      if (val < 0.0 || val > 2.0)
+      double val = arma::as_scalar(paretoSet.slice(solutionIdx));
+      if (!IsInBounds<double>(val, expectedLowerBound, expectedUpperBound))
       {
         allInRange = false;
         break;
@@ -126,8 +134,9 @@ TEST_CASE("NSGA2SchafferN1TestVectorBounds", "[NSGA2Test]")
 
 /**
  * Optimize for the Fonseca Fleming function using NSGA-II optimizer.
+ * Tests for data of type double.
  */
-TEST_CASE("NSGA2FonsecaFlemingTest", "[NSGA2Test]")
+TEST_CASE("NSGA2FonsecaFlemingDoubleTest", "[NSGA2Test]")
 {
   FonsecaFlemingFunction<arma::mat> FON;
   const double lowerBound = -4;
@@ -137,7 +146,7 @@ TEST_CASE("NSGA2FonsecaFlemingTest", "[NSGA2Test]")
   const double expectedLowerBound = -1.0 / sqrt(3);
   const double expectedUpperBound = 1.0 / sqrt(3);
 
-  NSGA2 opt(20, 4000, 0.6, 0.3, strength, tolerance, lowerBound, upperBound);
+  NSGA2 opt(20, 300, 0.6, 0.3, strength, tolerance, lowerBound, upperBound);
 
   typedef decltype(FON.objectiveA) ObjectiveTypeA;
   typedef decltype(FON.objectiveB) ObjectiveTypeB;
@@ -146,32 +155,34 @@ TEST_CASE("NSGA2FonsecaFlemingTest", "[NSGA2Test]")
   std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = FON.GetObjectives();
 
   opt.Optimize(objectives, coords);
-  std::vector<arma::mat> bestFront = opt.Front();
+  arma::cube paretoSet = opt.ParetoSet();
 
   bool allInRange = true;
 
-  for (size_t i = 0; i < bestFront.size(); i++)
+  for (size_t solutionIdx = 0; solutionIdx < paretoSet.n_slices; ++solutionIdx)
   {
-    const arma::mat solution = bestFront[i];
+    const arma::mat solution = paretoSet.slice(solutionIdx);
     double valX = arma::as_scalar(solution(0));
     double valY = arma::as_scalar(solution(1));
     double valZ = arma::as_scalar(solution(2));
 
-    if (!IsInBounds(valX, expectedLowerBound, expectedUpperBound) ||
-        !IsInBounds(valY, expectedLowerBound, expectedUpperBound) ||
-        !IsInBounds(valZ, expectedLowerBound, expectedUpperBound))
+    if (!IsInBounds<double>(valX, expectedLowerBound, expectedUpperBound) ||
+        !IsInBounds<double>(valY, expectedLowerBound, expectedUpperBound) ||
+        !IsInBounds<double>(valZ, expectedLowerBound, expectedUpperBound))
     {
       allInRange = false;
       break;
     }
   }
+
   REQUIRE(allInRange);
 }
 
 /**
  * Optimize for the Fonseca Fleming function using NSGA-II optimizer.
+ * Tests for data of type double.
  */
-TEST_CASE("NSGA2FonsecaFlemingTestVectorBounds", "[NSGA2Test]")
+TEST_CASE("NSGA2FonsecaFlemingTestVectorDoubleBounds", "[NSGA2Test]")
 {
   FonsecaFlemingFunction<arma::mat> FON;
   const arma::vec lowerBound = {-4, -4, -4};
@@ -181,7 +192,7 @@ TEST_CASE("NSGA2FonsecaFlemingTestVectorBounds", "[NSGA2Test]")
   const double expectedLowerBound = -1.0 / sqrt(3);
   const double expectedUpperBound = 1.0 / sqrt(3);
 
-  NSGA2 opt(20, 4000, 0.6, 0.3, strength, tolerance, lowerBound, upperBound);
+  NSGA2 opt(20, 300, 0.6, 0.3, strength, tolerance, lowerBound, upperBound);
 
   typedef decltype(FON.objectiveA) ObjectiveTypeA;
   typedef decltype(FON.objectiveB) ObjectiveTypeB;
@@ -190,24 +201,285 @@ TEST_CASE("NSGA2FonsecaFlemingTestVectorBounds", "[NSGA2Test]")
   std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = FON.GetObjectives();
 
   opt.Optimize(objectives, coords);
-  std::vector<arma::mat> bestFront = opt.Front();
+  arma::cube paretoSet = opt.ParetoSet();
 
   bool allInRange = true;
 
-  for (size_t i = 0; i < bestFront.size(); i++)
+  for (size_t solutionIdx = 0; solutionIdx < paretoSet.n_slices; ++solutionIdx)
   {
-    const arma::mat solution = bestFront[i];
+    const arma::mat solution = paretoSet.slice(solutionIdx);
     double valX = arma::as_scalar(solution(0));
     double valY = arma::as_scalar(solution(1));
     double valZ = arma::as_scalar(solution(2));
 
-    if (!IsInBounds(valX, expectedLowerBound, expectedUpperBound) ||
-        !IsInBounds(valY, expectedLowerBound, expectedUpperBound) ||
-        !IsInBounds(valZ, expectedLowerBound, expectedUpperBound))
+    if (!IsInBounds<double>(valX, expectedLowerBound, expectedUpperBound) ||
+        !IsInBounds<double>(valY, expectedLowerBound, expectedUpperBound) ||
+        !IsInBounds<double>(valZ, expectedLowerBound, expectedUpperBound))
     {
       allInRange = false;
       break;
     }
   }
+
   REQUIRE(allInRange);
 }
+
+/**
+ * Optimize for the Schaffer N.1 function using NSGA-II optimizer.
+ * Tests for data of type float.
+ */
+TEST_CASE("NSGA2SchafferN1FloatTest", "[NSGA2Test]")
+{
+  SchafferFunctionN1<arma::fmat> SCH;
+  const double lowerBound = -1000;
+  const double upperBound = 1000;
+  const double expectedLowerBound = 0.0;
+  const double expectedUpperBound = 2.0;
+
+  NSGA2 opt(20, 300, 0.5, 0.5, 1e-3, 1e-6, lowerBound, upperBound);
+
+  typedef decltype(SCH.objectiveA) ObjectiveTypeA;
+  typedef decltype(SCH.objectiveB) ObjectiveTypeB;
+
+  // We allow a few trials in case of poor convergence.
+  bool success = false;
+  for (size_t trial = 0; trial < 3; ++trial)
+  {
+    arma::fmat coords = SCH.GetInitialPoint();
+    std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = SCH.GetObjectives();
+
+    opt.Optimize(objectives, coords);
+    arma::fcube paretoSet = arma::conv_to<arma::fcube>::from(opt.ParetoSet());
+
+    bool allInRange = true;
+
+    for (size_t solutionIdx = 0; solutionIdx < paretoSet.n_slices; ++solutionIdx)
+    {
+      float val = arma::as_scalar(paretoSet.slice(solutionIdx));
+      if (!IsInBounds<float>(val, expectedLowerBound, expectedUpperBound))
+      {
+        allInRange = false;
+        break;
+      }
+    }
+
+    if (allInRange)
+    {
+      success = true;
+      break;
+    }
+  }
+
+  REQUIRE(success == true);
+}
+
+/**
+ * Optimize for the Schaffer N.1 function using NSGA-II optimizer.
+ * Tests for data of type float.
+ */
+TEST_CASE("NSGA2SchafferN1TestVectorFloatBounds", "[NSGA2Test]")
+{
+  // This test can be a little flaky, so we try it a few times.
+  SchafferFunctionN1<arma::fmat> SCH;
+  const arma::vec lowerBound = {-1000};
+  const arma::vec upperBound = {1000};
+  const double expectedLowerBound = 0.0;
+  const double expectedUpperBound = 2.0;
+
+  NSGA2 opt(20, 300, 0.5, 0.5, 1e-3, 1e-6, lowerBound, upperBound);
+
+  typedef decltype(SCH.objectiveA) ObjectiveTypeA;
+  typedef decltype(SCH.objectiveB) ObjectiveTypeB;
+
+  bool success = false;
+  for (size_t trial = 0; trial < 3; ++trial)
+  {
+    arma::fmat coords = SCH.GetInitialPoint();
+    std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = SCH.GetObjectives();
+
+    opt.Optimize(objectives, coords);
+    arma::fcube paretoSet = arma::conv_to<arma::fcube>::from(opt.ParetoSet());
+
+    bool allInRange = true;
+
+    for (size_t solutionIdx = 0; solutionIdx < paretoSet.n_slices; ++solutionIdx)
+    {
+      float val = arma::as_scalar(paretoSet.slice(solutionIdx));
+      if (!IsInBounds<float>(val, expectedLowerBound, expectedUpperBound))
+      {
+        allInRange = false;
+        break;
+      }
+    }
+
+    if (allInRange)
+    {
+      success = true;
+      break;
+    }
+  }
+
+  REQUIRE(success == true);
+}
+
+/**
+ * Optimize for the Fonseca Fleming function using NSGA-II optimizer.
+ * Tests for data of type float.
+ */
+TEST_CASE("NSGA2FonsecaFlemingFloatTest", "[NSGA2Test]")
+{
+  FonsecaFlemingFunction<arma::fmat> FON;
+  const double lowerBound = -4;
+  const double upperBound = 4;
+  const double tolerance = 1e-6;
+  const double strength = 1e-4;
+  const float expectedLowerBound = -1.0 / sqrt(3);
+  const float expectedUpperBound = 1.0 / sqrt(3);
+
+  NSGA2 opt(20, 300, 0.6, 0.3, strength, tolerance, lowerBound, upperBound);
+
+  typedef decltype(FON.objectiveA) ObjectiveTypeA;
+  typedef decltype(FON.objectiveB) ObjectiveTypeB;
+
+  arma::fmat coords = FON.GetInitialPoint();
+  std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = FON.GetObjectives();
+
+  opt.Optimize(objectives, coords);
+  arma::fcube paretoSet = arma::conv_to<arma::fcube>::from(opt.ParetoSet());
+
+  bool allInRange = true;
+
+  for (size_t solutionIdx = 0; solutionIdx < paretoSet.n_slices; ++solutionIdx)
+  {
+    const arma::fmat solution = paretoSet.slice(solutionIdx);
+    float valX = arma::as_scalar(solution(0));
+    float valY = arma::as_scalar(solution(1));
+    float valZ = arma::as_scalar(solution(2));
+
+    if (!IsInBounds<float>(valX, expectedLowerBound, expectedUpperBound) ||
+        !IsInBounds<float>(valY, expectedLowerBound, expectedUpperBound) ||
+        !IsInBounds<float>(valZ, expectedLowerBound, expectedUpperBound))
+    {
+      allInRange = false;
+      break;
+    }
+  }
+
+  REQUIRE(allInRange);
+}
+
+/**
+ * Optimize for the Fonseca Fleming function using NSGA-II optimizer.
+ * Tests for data of type float.
+ */
+TEST_CASE("NSGA2FonsecaFlemingTestVectorFloatBounds", "[NSGA2Test]")
+{
+  FonsecaFlemingFunction<arma::fmat> FON;
+  const arma::vec lowerBound = {-4, -4, -4};
+  const arma::vec upperBound = {4, 4, 4};
+  const double tolerance = 1e-6;
+  const double strength = 1e-4;
+  const float expectedLowerBound = -1.0 / sqrt(3);
+  const float expectedUpperBound = 1.0 / sqrt(3);
+
+  NSGA2 opt(20, 300, 0.6, 0.3, strength, tolerance, lowerBound, upperBound);
+
+  typedef decltype(FON.objectiveA) ObjectiveTypeA;
+  typedef decltype(FON.objectiveB) ObjectiveTypeB;
+
+  arma::fmat coords = FON.GetInitialPoint();
+  std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = FON.GetObjectives();
+
+  opt.Optimize(objectives, coords);
+  arma::fcube paretoSet = arma::conv_to<arma::fcube>::from(opt.ParetoSet());
+
+  bool allInRange = true;
+
+  for (size_t solutionIdx = 0; solutionIdx < paretoSet.n_slices; ++solutionIdx)
+  {
+    const arma::fmat solution = paretoSet.slice(solutionIdx);
+    float valX = arma::as_scalar(solution(0));
+    float valY = arma::as_scalar(solution(1));
+    float valZ = arma::as_scalar(solution(2));
+
+    if (!IsInBounds<float>(valX, expectedLowerBound, expectedUpperBound) ||
+        !IsInBounds<float>(valY, expectedLowerBound, expectedUpperBound) ||
+        !IsInBounds<float>(valZ, expectedLowerBound, expectedUpperBound))
+    {
+      allInRange = false;
+      break;
+    }
+  }
+
+  REQUIRE(allInRange);
+}
+
+/**
+ * Test against the first problem of ZDT Test Suite.  ZDT-1 is a 30 
+ * variable-2 objective problem with a convex Pareto Front.
+ * 
+ * NOTE: For the sake of runtime, only ZDT-1 is tested against the
+ * algorithm. Others have been tested separately.
+ */
+TEST_CASE("NSGA2ZDTONETest", "[NSGA2Test]")
+{
+  //! Parameters taken from original ZDT Paper.
+  ZDT1<> ZDT_ONE(100);
+  const double lowerBound = 0;
+  const double upperBound = 1;
+  const double tolerance = 1e-6;
+  const double mutationRate = 1e-2;
+  const double crossoverRate = 0.8;
+  const double strength = 1e-4;
+
+  NSGA2 opt(100, 250, crossoverRate, mutationRate, strength,
+    tolerance, lowerBound, upperBound);
+
+  typedef decltype(ZDT_ONE.objectiveF1) ObjectiveTypeA;
+  typedef decltype(ZDT_ONE.objectiveF2) ObjectiveTypeB;
+
+  arma::mat coords = ZDT_ONE.GetInitialPoint();
+  std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = ZDT_ONE.GetObjectives();
+
+  opt.Optimize(objectives, coords);
+
+  //! Refer the ZDT_ONE implementation for g objective implementation.
+  //! The optimal g value is taken from the docs of ZDT_ONE.
+  size_t numVariables = coords.size();
+  double sum = arma::accu(coords(arma::span(1, numVariables - 1), 0));
+  double g = 1. + 9. * sum / (static_cast<double>(numVariables - 1));
+
+  REQUIRE(g == Approx(1.0).margin(0.99));
+}
+
+/**
+ * Ensure that the reverse-compatible Front() function works.
+ *
+ * This test can be removed when Front() is removed, in ensmallen 3.x.
+ */
+TEST_CASE("NSGA2FrontTest", "[NSGA2Test]")
+{
+  SchafferFunctionN1<arma::mat> SCH;
+  const double lowerBound = -1000;
+  const double upperBound = 1000;
+
+  NSGA2 opt(20, 300, 0.5, 0.5, 1e-3, 1e-6, lowerBound, upperBound);
+
+  typedef decltype(SCH.objectiveA) ObjectiveTypeA;
+  typedef decltype(SCH.objectiveB) ObjectiveTypeB;
+
+  arma::mat coords = SCH.GetInitialPoint();
+  std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = SCH.GetObjectives();
+
+  opt.Optimize(objectives, coords);
+  arma::cube paretoFront = opt.ParetoFront();
+
+  std::vector<arma::mat> rcFront = opt.Front();
+
+  REQUIRE(paretoFront.n_slices == rcFront.size());
+  for (size_t i = 0; i < paretoFront.n_slices; ++i)
+  {
+    arma::mat paretoM = paretoFront.slice(i);
+    CheckMatrices(paretoM, rcFront[i]);
+  }
+}

tests/scd_test.cpp

@@ -196,7 +196,7 @@ TEST_CASE("SoftmaxRegressionFunctionPartialGradientTest", "[SCDTest]")
 
   // Create random class labels.
   arma::Row<size_t> labels = arma::randi<arma::Row<size_t> >(
-      points, arma::distr_param(0, numClasses));
+      points, arma::distr_param(0, numClasses - 1));
 
   // 2 objects for 2 terms in the cost function. Each term contributes towards
   // the gradient and thus need to be checked independently.

tests/sgd_test.cpp

@@ -70,6 +70,7 @@ TEST_CASE("GeneralizedRosenbrockTestFloat", "[SGDTest]")
       REQUIRE(result == Approx(0.0).margin(1e-5));
       for (size_t j = 0; j < i; ++j)
         REQUIRE(coordinates(j) == Approx(1.0).epsilon(1e-3));
+      break;
     }
   }
 }

tests/yogi_test.cpp

@@ -0,0 +1,74 @@
+/**
+ * @file yogi_test.cpp
+ * @author Marcus Edel
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#include <ensmallen.hpp>
+#include "catch.hpp"
+#include "test_function_tools.hpp"
+
+using namespace ens;
+using namespace ens::test;
+
+/**
+ * Test the Yogi optimizer on the Sphere function.
+ */
+TEST_CASE("YogiSphereFunctionTest", "[YogiTest]")
+{
+  SphereFunction f(2);
+  Yogi optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false);
+
+  arma::mat coordinates = f.GetInitialPoint();
+  optimizer.Optimize(f, coordinates);
+
+  REQUIRE(coordinates(0) == Approx(0.0).margin(0.1));
+  REQUIRE(coordinates(1) == Approx(0.0).margin(0.1));
+}
+
+/**
+ * Test the Yogi optimizer on the Sphere function with arma::fmat.
+ */
+TEST_CASE("YogiSphereFunctionTestFMat", "[YogiTest]")
+{
+  SphereFunction f(2);
+  Yogi optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false);
+
+  arma::fmat coordinates = f.GetInitialPoint<arma::fmat>();
+  optimizer.Optimize(f, coordinates);
+
+  REQUIRE(coordinates(0) == Approx(0.0).margin(0.1));
+  REQUIRE(coordinates(1) == Approx(0.0).margin(0.1));
+}
+
+/**
+ * Test the Yogi optimizer on the McCormick function.
+ */
+TEST_CASE("YogiMcCormickFunctionTest", "[YogiTest]")
+{
+  Yogi optimizer(0.5, 1, 0.7, 0.999, 1e-8, 500000, 1e-5, false);
+  FunctionTest<McCormickFunction>(optimizer, 0.5, 0.1);
+}
+
+/**
+ * Run Yogi on logistic regression and make sure the results are acceptable.
+ */
+TEST_CASE("YogiLogisticRegressionTest", "[YogiTest]")
+{
+  Yogi optimizer;
+  LogisticRegressionFunctionTest(optimizer, 0.003, 0.006);
+}
+
+/**
+ * Run Yogi on logistic regression and make sure the results are acceptable,
+ * using arma::fmat.
+ */
+TEST_CASE("YogiLogisticRegressionFMatTest", "[YogiTest]")
+{
+  Yogi optimizer;
+  LogisticRegressionFunctionTest<arma::fmat>(optimizer, 0.003, 0.006);
+}