Update and release version 2.22.0.

IPOP and BIPOP restart mechanisms (#403 )
Merge pull request #409 from mlpack/feature/arma_14.2.0_change
2024-11-29 13:51:28 -05:00 · 2024-11-29 03:25:55 +01:00 · 2024-11-20 14:58:03 -06:00 · 2024-11-19 14:02:16 -06:00 · 2024-11-18 08:17:10 -06:00 · 2024-11-18 08:06:30 -06:00
166 changed files with 15229 additions and 1379 deletions
@@ -1,13 +1,9 @@
 environment:
-  ARMADILLO_DOWNLOAD: "http://ftp.fau.de/macports/distfiles/armadillo/armadillo-8.400.0.tar.xz"
+  ARMADILLO_DOWNLOAD: "https://sourceforge.net/projects/arma/files/armadillo-10.8.2.tar.xz"
  BLAS_LIBRARY: "%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a"
  BLAS_LIBRARY_DLL: "%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll"

  matrix:
-    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
-      VSVER: Visual Studio 14 2015 Win64
-      MSBUILD: C:\Program Files (x86)\MSBuild\14.0\bin\MSBuild.exe
-
    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
      VSVER: Visual Studio 15 2017 Win64
      MSBUILD: C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\MSBuild.exe
@@ -16,6 +12,10 @@ environment:
      VSVER: Visual Studio 16 2019
      MSBUILD: C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe

+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+      VSVER: Visual Studio 17 2022
+      MSBUILD: C:\Program Files\Microsoft Visual Studio\2022\Community\MSBuild\Current\Bin\MSBuild.exe
+
 configuration: Release

 install:
@@ -26,7 +26,7 @@ build_script:
  - cd ..
  - appveyor DownloadFile %ARMADILLO_DOWNLOAD% -FileName armadillo.tar.xz
  - 7z x armadillo.tar.xz -so -txz | 7z x -si -ttar > nul
-  - cd armadillo-8.400.0 && mkdir build && cd build
+  - cd armadillo-10.8.2 && mkdir build && cd build
  - >
    cmake -G "%VSVER%"
    -DBLAS_LIBRARY:FILEPATH=%BLAS_LIBRARY%
@@ -43,14 +43,14 @@ build_script:
  - cd ensmallen && mkdir build && cd build
  - >
    cmake -G "%VSVER%"
-    -DARMADILLO_INCLUDE_DIR=%APPVEYOR_BUILD_FOLDER%/../armadillo-8.400.0/include/
+    -DARMADILLO_INCLUDE_DIR=%APPVEYOR_BUILD_FOLDER%/../armadillo-10.8.2/include/
    -DARMADILLO_LIBRARIES=%BLAS_LIBRARY%
    -DLAPACK_LIBRARY=%BLAS_LIBRARY%
    -DBLAS_LIBRARY=%BLAS_LIBRARY%
    -DCMAKE_BUILD_TYPE=Release ..
  - >
    "%MSBUILD%" "ensmallen.sln"
-    /m /verbosity:minimal /nologo /p:BuildInParallel=true
+    /m /verbosity:minimal /nologo /p:BuildInParallel=false

  # Run tests after copying libraries.
  - ps: cp C:\projects\ensmallen\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.* C:\projects\ensmallen\build\
@@ -0,0 +1,22 @@
+# Once a PR has been approved by one member of the mlpack organization, a second
+# approving review will automatically be added 24 hours later.  This allows time
+# for other maintainers to take a look.
+name: Auto-approve pull requests
+on:
+  schedule:
+   # Run roughly every four hours.
+   - cron: "15 0,4,8,12,16,20 * * *"
+
+jobs:
+  auto-approve:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+
+    steps:
+     - name: Auto-approve pull requests
+       uses: rcurtin/actions/auto-approve@v1
+       with:
+         repo-token: ${{ secrets.GITHUB_TOKEN }}
+         approval-message:
+           'Second approval provided automatically after 24 hours. :+1:'
@@ -0,0 +1,24 @@
+name: Close inactive issues
+on:
+  schedule:
+    - cron: "30 1 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v9
+        with:
+          days-before-issues-stale: 30
+          days-before-issue-close: 7
+          stale-issue-label: "s: stale"
+          stale-pr-label: "s: stale"
+          stale-issue-message: "This issue has been automatically marked as stale because it has not had any recent activity.  It will be closed in 7 days if no further activity occurs.  Thank you for your contributions! :+1:"
+          days-before-pr-stale: 30,
+          days-before-pr-close: 14
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          exempt-issue-labels: "s: keep open"
+          exempt-pr-labels: "s: keep open"
@@ -0,0 +1,17 @@
+# Post a message to new contributors that they can get some stickers mailed to
+# them.
+name: 'Stickers for new contributors'
+on:
+  pull_request:
+    types: [closed]
+
+jobs:
+  sticker_comment:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.merged == true
+    steps:
+        # Forked version of first-interaction that runs only on first merged PR.
+      - uses: rcurtin/actions/stickers@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          pr-message: "Hello there!  Thanks for your contribution.  Congratulations on your first contribution to mlpack!  If you'd like to add your name to the list of contributors in `COPYRIGHT.txt` and you haven't already, please feel free to push a change to this PR---or, if it gets merged before you can, feel free to open another PR.\n\nIn addition, if you'd like some stickers to put on your laptop, we can get them in the mail for you.  Just send an email with your physical mailing address to stickers@mlpack.org, and then one of the mlpack maintainers will put some stickers in an envelope for you.  It may take a few weeks to get them, depending on your location. :+1:"
@@ -1,5 +1,5 @@
 os: linux
-dist: trusty
+dist: focal
 language: cpp

 env:
@@ -23,10 +23,10 @@ script:
  - if [ $ARMADILLO == "latest" ]; then
      curl https://ftp.fau.de/macports/distfiles/armadillo/`curl https://ftp.fau.de/macports/distfiles/armadillo/ -- | grep '.tar.xz' | sed 's/^.*<a href="\(armadillo-[0-9]*.[0-9]*.[0-9]*.tar.xz\)".*$/\1/' | tail -1` | tar xvJ && cd armadillo*;
    else
-      curl https://ftp.fau.de/macports/distfiles/armadillo/armadillo-8.400.0.tar.xz | tar -xvJ && cd armadillo*;
+      curl -L https://sourceforge.net/projects/arma/files/armadillo-10.8.2.tar.xz | tar -xvJ && cd armadillo*;
    fi
  - cmake . && make && sudo make install && cd ..
-  - mkdir build && cd build && cmake -DCMAKE_CXX_FLAGS="-Werror" -DCMAKE_C_FLAGS="-Werror" .. && make -j2
+  - mkdir build && cd build && cmake .. && make ensmallen_tests -j2
  - CTEST_OUTPUT_ON_FAILURE=1 travis_wait 30 ctest -j2

 notifications:
@@ -1,18 +1,20 @@
 # ensmallen CMake configuration.  This project has no configurable options---it
 # just installs the headers to the install location, and optionally builds the
 # test program.
-cmake_minimum_required(VERSION 3.3.2)
-project(ensmallen 
+cmake_minimum_required(VERSION 3.3.2...3.5)
+if (NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE "Release")  # ensure the tests are built with optimisation
+endif ()
+project(ensmallen
        LANGUAGES C CXX)

 # Configurable options for CMake.
 option(USE_OPENMP "If available, use OpenMP for parallelization." ON)
-option(BUILD_TESTS "Build tests." ON)

 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/CMake")

-# Set required C++ standard to C++11.
-set(CMAKE_CXX_STANDARD 11)
+# Set required C++ standard to C++14.
+set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)

 # Extract version from sources.
@@ -58,7 +60,7 @@ if(USE_OPENMP)
 endif()

 # Find Armadillo and link it.
-find_package(Armadillo 8.400.0 REQUIRED)
+find_package(Armadillo 10.8.2 REQUIRED)
 target_link_libraries(ensmallen INTERFACE Armadillo::Armadillo)

 # Set helper variables for creating the version, config and target files.
@@ -96,6 +98,4 @@ install(FILES ${CMAKE_SOURCE_DIR}/include/ensmallen.hpp

 # Enable testing and build tests.
 enable_testing()
-if (BUILD_TESTS)
-  add_subdirectory(tests)
-endif()
+add_subdirectory(tests)
@@ -108,8 +108,8 @@ $ cd ensmallen

 # - or -

-$ wget http://ensmallen.org/files/ensmallen-2.16.2.tar.gz
-$ tar -xvzpf ensmallen-2.16.2.tar.gz
+$ wget http://ensmallen.org/files/ensmallen-2.22.0.tar.gz
+$ tar -xvzpf ensmallen-2.22.0.tar.gz
 $ cd ensmallen-latest
 ```

@@ -7,10 +7,10 @@ Source:

 Files: *
 Copyright:
-  Copyright 2008-2018, Ryan Curtin <ryan@ratml.org>
+  Copyright 2008-2023, Ryan Curtin <ryan@ratml.org>
  Copyright 2008-2012, Dongryeol Lee <dongryel@cc.gatech.edu>
  Copyright 2010-2012, James Cline <james.cline@gatech.edu>
-  Copyright 2013-2018, Marcus Edel <marcus.edel@fu-berlin.de>
+  Copyright 2013-2023, Marcus Edel <marcus.edel@fu-berlin.de>
  Copyright 2013-2018, Sumedh Ghaisas <sumedhghaisas@gmail.com>
  Copyright 2013, Mudit Raj Gupta <mudit.raaj.gupta@gmail.com>
  Copyright 2014, Ryan Birmingham <birm@gatech.edu>
@@ -33,7 +33,7 @@ Copyright:
  Copyright 2018, B Kartheek Reddy <bkartheekreddy@gmail.com>
  Copyright 2018, Moksh Jain <mokshjn00@gmail.com>
  Copyright 2018, Shikhar Jaiswal <jaiswalshikhar87@gmail.com>
-  Copyright 2018, Conrad Sanderson
+  Copyright 2018-2023, Conrad Sanderson
  Copyright 2018, Dan Timson
  Copyright 2019, Rahul Ganesh Prabhu
  Copyright 2019, Roberto Hueso <robertohueso96@gmail.com>
@@ -1,3 +1,155 @@
+### ensmallen 2.22.0: "E-Bike Excitement"
+###### 2024-11-29
+ * Update to C++14 standard
+   ([#400](https://github.com/mlpack/ensmallen/pull/400)).
+
+ * Bump minimum Armadillo version to 10.8
+   ([#404](https://github.com/mlpack/ensmallen/pull/404)).
+
+ * For Armadillo 14.2.0 switch to `.index_min()` and `.index_max()`
+   ([#409](https://github.com/mlpack/ensmallen/pull/409)).
+
+ * Added IPOP and BIPOP restart mechanisms for CMA-ES.
+   ([#403](https://github.com/mlpack/ensmallen/pull/403)).
+
+
+### ensmallen 2.21.1: "Bent Antenna"
+###### 2024-02-15
+ * Fix numerical precision issues for small-gradient L-BFGS scaling factor
+   computations ([#392](https://github.com/mlpack/ensmallen/pull/392)).
+
+ * Ensure the tests are built with optimisation enabled
+   ([#394](https://github.com/mlpack/ensmallen/pull/394)).
+
+### ensmallen 2.21.0: "Bent Antenna"
+###### 2023-11-27
+ * Clarify return values for different callback types
+   ([#383](https://github.com/mlpack/ensmallen/pull/383)).
+
+ * Fix return types of callbacks
+   ([#382](https://github.com/mlpack/ensmallen/pull/382)).
+
+ * Minor cleanup for printing optimization reports via `Report()`
+   ([#385](https://github.com/mlpack/ensmallen/pull/385)).
+
+### ensmallen 2.20.0: "Stripped Bolt Head"
+###### 2023-10-02
+ * Implementation of Active CMAES
+   ([#367](https://github.com/mlpack/ensmallen/pull/367)).
+
+ * LBFGS: avoid generation of NaNs, and add checks for finite values
+   ([#368](https://github.com/mlpack/ensmallen/pull/368)).
+
+ * Fix CNE test tolerances
+   ([#360](https://github.com/mlpack/ensmallen/pull/360)).
+
+ * Rename `SCD` optimizer, to `CD`
+   ([#379](https://github.com/mlpack/ensmallen/pull/379)).
+
+### ensmallen 2.19.1: "Eight Ball Deluxe"
+###### 2023-01-30
+ * Avoid deprecation warnings in Armadillo 11.2+
+   ([#347](https://github.com/mlpack/ensmallen/pull/347)).
+
+### ensmallen 2.19.0: "Eight Ball Deluxe"
+###### 2022-04-06
+ * Added DemonSGD and DemonAdam optimizers
+   ([#211](https://github.com/mlpack/ensmallen/pull/211)).
+
+ * Fix bug with Adam-like optimizers not resetting when `resetPolicy` is `true`.
+   ([#340](https://github.com/mlpack/ensmallen/pull/340)).
+
+ * Add Yogi optimizer
+   ([#232](https://github.com/mlpack/ensmallen/pull/232)).
+
+ * Add AdaBelief optimizer
+   ([#233](https://github.com/mlpack/ensmallen/pull/233)).
+
+ * Add AdaSqrt optimizer
+   ([#234](https://github.com/mlpack/ensmallen/pull/234)).
+
+ * Bump check for minimum supported version of Armadillo
+   ([#342](https://github.com/mlpack/ensmallen/pull/342)).
+
+### ensmallen 2.18.2: "Fairmount Bagel"
+###### 2022-02-13
+ * Update Catch2 to 2.13.8
+  ([#336](https://github.com/mlpack/ensmallen/pull/336)).
+
+ * Fix epoch timing output
+  ([#337](https://github.com/mlpack/ensmallen/pull/337)).
+
+### ensmallen 2.18.1: "Fairmount Bagel"
+###### 2021-11-19
+ * Accelerate SGD test time
+   ([#330](https://github.com/mlpack/ensmallen/pull/300)).
+
+ * Fix potential infinite loop in CMAES
+   ([#331](https://github.com/mlpack/ensmallen/pull/331)).
+
+ * Fix SCD partial gradient test
+   ([#332](https://github.com/mlpack/ensmallen/pull/332)).
+
+### ensmallen 2.18.0: "Fairmount Bagel"
+###### 2021-10-20
+ * Add gradient value clipping and gradient norm scaling callback
+   ([#315](https://github.com/mlpack/ensmallen/pull/315)).
+
+ * Remove superfluous CMake option to build the tests
+   ([#313](https://github.com/mlpack/ensmallen/pull/313)).
+
+ * Bump minimum Armadillo version to 9.800
+   ([#318](https://github.com/mlpack/ensmallen/pull/318)).
+
+ * Update Catch2 to 2.13.7
+   ([#322](https://github.com/mlpack/ensmallen/pull/322)).
+
+ * Remove redundant template argument for C++20 compatibility
+   ([#324](https://github.com/mlpack/ensmallen/pull/324)).
+
+ * Fix MOEAD test stability
+   ([#327](https://github.com/mlpack/ensmallen/pull/327)).
+
+### ensmallen 2.17.0: "Pachis Din Me Pesa Double"
+###### 2021-07-06
+ * CheckArbitraryFunctionTypeAPI extended for MOO support
+   ([#283](https://github.com/mlpack/ensmallen/pull/283)).
+
+ * Refactor NSGA2
+   ([#263](https://github.com/mlpack/ensmallen/pull/263),
+   [#304](https://github.com/mlpack/ensmallen/pull/304)).
+
+ * Add Indicators for Multiobjective optimizers
+   ([#285](https://github.com/mlpack/ensmallen/pull/285)).
+
+ * Make Callback flexible for MultiObjective Optimizers
+   ([#289](https://github.com/mlpack/ensmallen/pull/289)).
+
+ * Add ZDT Test Suite
+   ([#273](https://github.com/mlpack/ensmallen/pull/273)).
+
+ * Add MOEA-D/DE Optimizer
+   ([#269](https://github.com/mlpack/ensmallen/pull/269)).
+
+ * Introduce Policy Methods for MOEA/D-DE
+   ([#293](https://github.com/mlpack/ensmallen/pull/293)).
+
+ * Add Das-Dennis weight initialization method
+   ([#295](https://github.com/mlpack/ensmallen/pull/295)).
+
+ * Add Dirichlet Weight Initialization
+   ([#296](https://github.com/mlpack/ensmallen/pull/296)).
+
+ * Improved installation and compilation instructions
+   ([#300](https://github.com/mlpack/ensmallen/pull/300)).
+
+ * Disable building the tests by default for faster installation
+   ([#303](https://github.com/mlpack/ensmallen/pull/303)).
+
+ * Modify matrix initialisation to take into account
+   default element zeroing in Armadillo 10.5
+   ([#305](https://github.com/mlpack/ensmallen/pull/305)).
+
 ### ensmallen 2.16.2: "Severely Dented Can Of Polyurethane"
 ###### 2021-03-24
 * Fix CNE test trials
@@ -2,33 +2,35 @@
  <a href="http://ensmallen.org/"><img src="http://ensmallen.org/img/ensmallen_text.svg" style="background-color:rgba(0,0,0,0);" height=230 alt="ensmallen: a C++ header-only library for numerical optimization"></a>
 </h2>

-**ensmallen** is a C++ header-only library for numerical optimization.
+**ensmallen** is a high-quality C++ library for non-linear numerical optimization.

-Documentation and downloads: http://ensmallen.org
-
-ensmallen provides a simple set of abstractions for writing an objective
-function to optimize. It also provides a large set of standard and cutting-edge
-optimizers that can be used for virtually any numerical optimization task.
-These include full-batch gradient descent techniques, small-batch techniques,
-gradient-free optimizers, and constrained optimization.
+ensmallen provides many types of optimizers that can be used
+for virtually any numerical optimization task.
+This includes gradient descent techniques, gradient-free optimizers,
+and constrained optimization.
+Examples include L-BFGS, SGD, CMAES and Simulated Annealing.
+ensmallen also allows optional callbacks to customize the optimization process.

+Documentation and downloads: https://ensmallen.org

 ### Requirements

-* C++ compiler with C++11 support
-* Armadillo: http://arma.sourceforge.net
+* C++ compiler with C++14 support
+* Armadillo: https://arma.sourceforge.net
 * OpenBLAS or Intel MKL or LAPACK (see Armadillo site for details)


 ### Installation

-ensmallen can be installed with CMake 3.3 or later.
-If CMake is not already available on your system, it can be obtained from https://cmake.org
+ensmallen can be installed in several ways: either manually or via cmake, 
+with or without root access.

-If you are using an older system such as RHEL 7 or CentOS 7,
-an updated version of CMake is also available via the EPEL repository via the `cmake3` package.
+The cmake based installation will check the requirements 
+and optionally build the tests. If cmake 3.3 (or a later version) 
+is not already available on your system, it can be obtained 
+from [cmake.org](https://cmake.org).

-Example installation:
+Example cmake based installation with root access:

 ```
 mkdir build
@@ -37,10 +39,54 @@ cmake ..
 sudo make install
 ```

+Example cmake based installation without root access, 
+installing into `/home/blah/` (adapt as required):

-### Example Usage
+```
+mkdir build
+cd build
+cmake .. -DCMAKE_INSTALL_PREFIX:PATH=/home/blah/
+make install
+```

-See [`example.cpp`](example.cpp) for example usage of the L-BFGS optimizer in a linear regression setting.
+The above will create a directory named `/home/blah/include/` 
+and place all ensmallen headers there.
+
+To optionally build and run the tests
+(after running cmake as above),
+use the following additional commands:
+
+```
+make ensmallen_tests
+./ensmallen_tests --durations yes
+```
+
+Manual installation involves simply copying the `include/ensmallen.hpp` header 
+***and*** the associated `include/ensmallen_bits` directory to a location 
+such as `/usr/include/` which is searched by your C++ compiler.
+If you can't use `sudo` or don't have write access to `/usr/include/`, 
+use a directory within your own home directory (eg. `/home/blah/include/`).
+
+
+### Example Compilation
+
+If you have installed ensmallen in a standard location such as `/usr/include/`:
+
+    g++ prog.cpp -o prog -O2 -larmadillo
+    
+If you have installed ensmallen in a non-standard location, 
+such as `/home/blah/include/`, you will need to make sure 
+that your C++ compiler searches `/home/blah/include/` 
+by explicitly specifying the directory as an argument/option. 
+For example, using the `-I` switch in gcc and clang:
+
+    g++ prog.cpp -o prog -O2 -I /home/blah/include/ -larmadillo
+
+
+### Example Optimization
+
+See [`example.cpp`](example.cpp) for example usage of the L-BFGS optimizer 
+in a linear regression setting.


 ### License
@@ -48,7 +94,7 @@ See [`example.cpp`](example.cpp) for example usage of the L-BFGS optimizer in a
 Unless stated otherwise, the source code for **ensmallen** is licensed under the
 3-clause BSD license (the "License").  A copy of the License is included in the
 "LICENSE.txt" file.  You may also obtain a copy of the License at
-http://opensource.org/licenses/BSD-3-Clause .
+http://opensource.org/licenses/BSD-3-Clause


 ### Citation
@@ -57,28 +103,20 @@ Please cite the following paper if you use ensmallen in your research and/or
 software. Citations are useful for the continued development and maintenance of
 the library.

-* S. Bhardwaj, R. Curtin, M. Edel, Y. Mentekidis, C. Sanderson.  
-  [ensmallen: a flexible C++ library for efficient function optimization](http://www.ensmallen.org/files/ensmallen_2018.pdf).  
-  Workshop on Systems for ML and Open Source Software at NIPS 2018.
+* Ryan R. Curtin, Marcus Edel, Rahul Ganesh Prabhu, Suryoday Basak, Zhihao Lou, Conrad Sanderson.  
+  [The ensmallen library for flexible numerical optimization](https://jmlr.org/papers/volume22/20-416/20-416.pdf).  
+  Journal of Machine Learning Research, Vol. 22, No. 166, 2021.

 ```
-@article{DBLP:journals/corr/abs-1810-09361,
-  author    = {Shikhar Bhardwaj and
-               Ryan R. Curtin and
-               Marcus Edel and
-               Yannis Mentekidis and
-               Conrad Sanderson},
-  title     = {ensmallen: a flexible {C++} library for efficient function optimization},
-  journal   = {CoRR},
-  volume    = {abs/1810.09361},
-  doi       = {10.5281/zenodo.2008650},
-  year      = {2018},
-  url       = {http://arxiv.org/abs/1810.09361},
-  archivePrefix = {arXiv},
-  eprint    = {1810.09361},
-  timestamp = {Wed, 31 Oct 2018 14:24:29 +0100},
-  biburl    = {https://dblp.org/rec/bib/journals/corr/abs-1810-09361},
-  bibsource = {dblp computer science bibliography, https://dblp.org}
+@article{ensmallen_JMLR_2021,
+  author  = {Ryan R. Curtin and Marcus Edel and Rahul Ganesh Prabhu and Suryoday Basak and Zhihao Lou and Conrad Sanderson},
+  title   = {The ensmallen library for flexible numerical optimization},
+  journal = {Journal of Machine Learning Research},
+  year    = {2021},
+  volume  = {22},
+  number  = {166},
+  pages   = {1--6},
+  url     = {http://jmlr.org/papers/v22/20-416.html}
 }
 ```

@@ -111,3 +149,4 @@ the library.
 * N Rajiv Vaidyanathan
 * Roberto Hueso
 * Sayan Goswami
+
@@ -136,7 +136,7 @@ EarlyStopAtMinLoss cb(
      // You could also, e.g., print the validation loss here to watch it converge.
      return lrfValidation.Evaluate(coordinates);
    });
-    
+
 arma::mat coordinates = lrfTrain.GetInitialPoint();
 SMORMS3 smorms3;
 smorms3.Optimize(lrfTrain, coordinates, cb);
@@ -144,6 +144,82 @@ smorms3.Optimize(lrfTrain, coordinates, cb);

 </details>

+### Gradient Clipping
+
+One challenge in optimization is dealing with "exploding gradients", where large
+parameter gradients can cause the optimizer to make excessively large updates,
+potentially pushing the model into regions of high loss or causing numerical
+instability. This can happen due to:
+
+* A high learning rate, leading to large gradient updates.
+* Poorly scaled datasets, resulting in significant variance between data points.
+* A loss function that generates disproportionately large error values.
+    
+Common solutions for this problem are:
+
+#### GradClipByNorm
+
+In this method, the solution is to change the derivative
+of the error before applying the update step.  One option is to clip the norm
+`||g||` of the gradient `g` before a parameter update. So given the gradient,
+and a maximum norm value, the callback normalizes the gradient so that its
+L2-norm is less than or equal to the given maximum norm value.
+
+#### Constructors
+
+ * `GradClipByNorm(`_`maxNorm`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`maxNorm`** | The maximum clipping value. | |
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+optimizer.Optimize(f, coordinates, GradClipByNorm(0.3));
+```
+
+#### GradClipByValue
+
+In this method, the solution is to change the derivative
+of the error before applying the update step.  One option is to clip the
+parameter gradient element-wise before a parameter update.
+
+#### Constructors
+
+ * `GradClipByValue(`_`min, max`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`min`** | The minimum value to clip to. | |
+| `double` | **`max`** | The maximum value to clip to. | |
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+optimizer.Optimize(f, coordinates, GradClipByValue(0, 1.3));
+```
+
 ### PrintLoss

 Callback that prints loss to stdout or a specified output stream.
@@ -208,7 +284,7 @@ optimizer.Optimize(f, coordinates, ProgressBar());

 </details>

-### Report 
+### Report

 Callback that prints a optimizer report to stdout or a specified output stream.

@@ -341,18 +417,18 @@ std::cout << "The optimized model found by AdaDelta has the "
 Callbacks are called at several states during the optimization process:

 * At the beginning and end of the optimization process.
-* After any call to `Evaluate()` and `EvaluateConstraint`.
-* After any call to `Gradient()` and `GradientConstraint`.
+* After any call to `Evaluate()` and `EvaluateConstraint()`.
+* After any call to `Gradient()` and `GradientConstraint()`.
 * At the start and end of an epoch.

-Each callback provides optimization relevant information that can be accessed or
+Each callback provides optimization-relevant information that can be accessed or
 modified.

 ### BeginOptimization

 Called at the beginning of the optimization process.

- * `BeginOptimization(`_`optimizer, function, coordinates`_`)`
+ * `void BeginOptimization(`_`optimizer, function, coordinates`_`)`

 #### Attributes

@@ -366,7 +442,7 @@ Called at the beginning of the optimization process.

 Called at the end of the optimization process.

- * `EndOptimization(`_`optimizer, function, coordinates`_`)`
+ * `void EndOptimization(`_`optimizer, function, coordinates`_`)`

 #### Attributes

@@ -380,7 +456,9 @@ Called at the end of the optimization process.

 Called after any call to `Evaluate()`.

- * `Evaluate(`_`optimizer, function, coordinates, objective`_`)`
+ * `bool Evaluate(`_`optimizer, function, coordinates, objective`_`)`
+
+If the callback returns `true`, the optimization will be terminated.

 #### Attributes

@@ -393,9 +471,11 @@ Called after any call to `Evaluate()`.

 ### EvaluateConstraint

- Called after any call to `EvaluateConstraint()`.
+Called after any call to `EvaluateConstraint()`.

- * `EvaluateConstraint(`_`optimizer, function, coordinates, constraint, constraintValue`_`)`
+ * `bool EvaluateConstraint(`_`optimizer, function, coordinates, constraint, constraintValue`_`)`
+
+If the callback returns `true`, the optimization will be terminated.

 #### Attributes

@@ -409,9 +489,11 @@ Called after any call to `Evaluate()`.

 ### Gradient

- Called after any call to `Gradient()`.
+Called after any call to `Gradient()`.

- * `Gradient(`_`optimizer, function, coordinates, gradient`_`)`
+ * `bool Gradient(`_`optimizer, function, coordinates, gradient`_`)`
+
+If the callback returns `true`, the optimization will be terminated.

 #### Attributes

@@ -424,9 +506,11 @@ Called after any call to `Evaluate()`.

 ### GradientConstraint

- Called after any call to `GradientConstraint()`.
+Called after any call to `GradientConstraint()`.

- * `GradientConstraint(`_`optimizer, function, coordinates, constraint, gradient`_`)`
+ * `bool GradientConstraint(`_`optimizer, function, coordinates, constraint, gradient`_`)`
+
+If the callback returns `true`, the optimization will be terminated.

 #### Attributes

@@ -443,7 +527,9 @@ Called after any call to `Evaluate()`.
 Called at the beginning of a pass over the data. The objective may be exact or
 an estimate depending on `exactObjective` value.

- * `BeginEpoch(`_`optimizer, function, coordinates, epoch, objective`_`)`
+ * `bool BeginEpoch(`_`optimizer, function, coordinates, epoch, objective`_`)`
+
+If the callback returns `true`, the optimization will be terminated.

 #### Attributes

@@ -460,7 +546,9 @@ an estimate depending on `exactObjective` value.
 Called at the end of a pass over the data. The objective may be exact or
 an estimate depending on `exactObjective` value.

- * `EndEpoch(`_`optimizer, function, coordinates, epoch, objective`_`)`
+ * `bool EndEpoch(`_`optimizer, function, coordinates, epoch, objective`_`)`
+
+If the callback returns `true`, the optimization will be terminated.

 #### Attributes

@@ -472,6 +560,25 @@ an estimate depending on `exactObjective` value.
 | `size_t` | **`epoch`** | The index of the current epoch. |
 | `double` | **`objective`** | Objective value of the current point. |

+### GenerationalStepTaken
+
+Called after the evolution of a single generation. Intended specifically for
+MultiObjective Optimizers.
+
+ * `bool GenerationalStepTaken(`_`optimizer, function, coordinates, objectives, frontIndices`_`)`
+
+If the callback returns `true`, the optimization will be terminated.
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+| `ObjectivesVecType` | **`objectives`** | The set of calculated objectives so far. |
+| `IndicesType` | **`frontIndices`** | The indices of the members belonging to Pareto Front. |
+
 ## Custom Callbacks

 ### Learning rate scheduling
@@ -515,7 +622,7 @@ class ExponentialDecay
  // Callback function called at the end of a pass over the data. We are only
  // interested in the current epoch and the optimizer, we ignore the rest.
  template<typename OptimizerType, typename FunctionType, typename MatType>
-  void EndEpoch(OptimizerType& optimizer,
+  bool EndEpoch(OptimizerType& optimizer,
                FunctionType& /* function */,
                const MatType& /* coordinates */,
                const size_t epoch,
@@ -524,6 +631,9 @@ class ExponentialDecay
    // Update the learning rate.
    optimizer.StepSize() = learningRate * (1.0 - std::pow(decay,
        (double) epoch));
+
+    // Do not terminate the optimization.
+    return false;
  }

  double learningRate;
@@ -595,7 +705,7 @@ class EarlyStop
  // the current objective. We are only interested in the objective and ignore
  // the rest.
  template<typename OptimizerType, typename FunctionType, typename MatType>
-  void EndEpoch(OptimizerType& /* optimizer */,
+  bool EndEpoch(OptimizerType& /* optimizer */,
                FunctionType& /* function */,
                const MatType& /* coordinates */,
                const size_t /* epoch */,
@@ -307,7 +307,7 @@ regular implementation of the `Gradient()`, so that function may be omitted.
 If these functions are implemented, the following partially differentiable
 function optimizers can be used:

- - [Stochastic Coordinate Descent](#stochastic-coordinate-descent-scd)
+ - [Coordinate Descent](#coordinate-descent-cd)

 ## Arbitrary separable functions

@@ -358,7 +358,10 @@ Each of the implemented methods is allowed to have additional cv-modifiers

 The following optimizers can be used with arbitrary separable functions:

- - [CMAES](#cmaes)
+ - [Active CMA-ES](#active-cma-es)
+ - [BIPOP CMA-ES](#bipop-cma-es)
+ - [CMA-ES](#cma-es)
+ - [IPOP CMA-ES](#ipop-cma-es)

 Each of these optimizers has an `Optimize()` function that is called as
 `Optimize(f, x)` where `f` is the function to be optimized and `x` holds the
@@ -453,7 +456,7 @@ int main()
  // parameters, so the shape is 10x1.
  arma::mat params(10, 1, arma::fill::randn);

-  // Use the CMAES optimizer with default parameters to minimize the
+  // Use the CMA-ES optimizer with default parameters to minimize the
  // LinearRegressionFunction.
  // The ens::CMAES type can be replaced with any suitable ensmallen optimizer
  // that can handle arbitrary separable functions.
@@ -461,7 +464,7 @@ int main()
  LinearRegressionFunction lrf(data, responses);
  cmaes.Optimize(lrf, params);

-  std::cout << "The optimized linear regression model found by CMAES has the "
+  std::cout << "The optimized linear regression model found by CMA-ES has the "
      << "parameters " << params.t();
 }
 ```
@@ -554,9 +557,11 @@ Each of the implemented methods is allowed to have additional cv-modifiers

 The following optimizers can be used with differentiable separable functions:

+ - [AdaBelief](#adabelief)
 - [AdaBound](#adabound)
 - [AdaDelta](#adadelta)
 - [AdaGrad](#adagrad)
+ - [AdaSqrt](#adasqrt)
 - [Adam](#adam)
 - [AdaMax](#adamax)
 - [AMSBound](#amsbound)
@@ -876,18 +881,104 @@ NSGA2 nsga;
 double bestFrontSum = nsga.Optimize(objectives, coordinates);

 // Set `bestFront` to contain all of the coordinates on the best front.
-std::vector<arma::mat> bestFront = optimizer.Front();
+arma::cube bestFront = optimizer.ParetoFront();
 }
 ```

 </details>

+### Performance Indicators
+
+Performance indicators in multiobjective optimization provide essential metrics 
+for evaluating solution quality, such as convergence to the Pareto front and solution 
+diversity.
+
+The ensmallen library offers three such indicators, aiding in the assessment and comparison 
+of different optimization methods:
+
+#### Epsilon
+
+Epsilon metric is a performance metric used in multi-objective optimization which measures 
+the smallest factor by which a set of solution objectives must be scaled to dominate a 
+reference set of solutions. Specifically, given a set of Pareto-optimal solutions, the 
+epsilon indicator finds the minimum value ϵ such that each solution in the set is at 
+least as good as every solution in the reference set when the objectives are scaled by ϵ.
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+arma::cube referenceFront(2, 1, 3);
+double tol = 1e-10;
+referenceFront.slice(0) = arma::vec{0.01010101, 0.89949622};
+referenceFront.slice(1) = arma::vec{0.02020202, 0.85786619};
+referenceFront.slice(2) = arma::vec{0.03030303, 0.82592234};
+arma::cube front = referenceFront * 1.1;
+// eps is approximately 1.1
+double eps = Epsilon::Evaluate(front, referenceFront);
+```
+</details>
+
+#### IGD
+
+Inverse Generational Distance (IGD) is a performance metric used in multi-objective optimization 
+to evaluate the quality of a set of solutions relative to a reference set, typically representing 
+the true Pareto front. IGD measures the average distance from each point in the reference set to 
+the closest point in the obtained solution set.
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+arma::cube referenceFront(2, 1, 3);
+double tol = 1e-10;
+referenceFront.slice(0) = arma::vec{0.01010101, 0.89949622};
+referenceFront.slice(1) = arma::vec{0.02020202, 0.85786619};
+referenceFront.slice(2) = arma::vec{0.03030303, 0.82592234};
+arma::cube front = referenceFront * 1.1;
+// The third parameter is the power constant in the distance formula.
+// IGD is approximately 0.05329
+double igd = IGD::Evaluate(front, referenceFront, 1);
+```
+</details>
+
+#### IGD Plus
+
+IGD Plus (IGD+) is a variant of the Inverse Generational Distance (IGD) metric used in multi-objective 
+optimization. It refines the traditional IGD metric by incorporating a preference for Pareto-dominance 
+in the distance calculation. This modification helps IGD+ better reflect both the convergence to the 
+Pareto front and the diversity of the solution set.
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+arma::cube referenceFront(2, 1, 3);
+double tol = 1e-10;
+referenceFront.slice(0) = arma::vec{0.01010101, 0.89949622};
+referenceFront.slice(1) = arma::vec{0.02020202, 0.85786619};
+referenceFront.slice(2) = arma::vec{0.03030303, 0.82592234};
+arma::cube front = referenceFront * 1.1;
+// IGDPlus is approximately 0.05329
+double igdPlus = IGDPlus::Evaluate(front, referenceFront);
+```
+</details>
+
 *Note*: all multi-objective function optimizers have both the function `Optimize()` to find the
-best front, and also the function `Front()` to return all sets of coordinates that are on the
+best front, and also the function `ParetoFront()` to return all sets of solutions that are on the
 front.

 The following optimizers can be used with multi-objective functions:
 - [NSGA2](#nsga2)
+- [MOEA/D-DE](#moead)
+- [AGEMOEA](#agemoea)
+
+#### See also:
+* [Performance Assessment of Multiobjective Optimizers: An Analysis and Review](https://sop.tik.ee.ethz.ch/publicationListFiles/ztlf2003a.pdf)
+* [Modified Distance Calculation in Generational Distance and Inverted Generational Distance](https://link.springer.com/chapter/10.1007/978-3-319-15892-1_8)

 ## Constrained functions

@@ -15,22 +15,27 @@
 #ifndef ENSMALLEN_HPP
 #define ENSMALLEN_HPP

-// certain compilers are way behind the curve
-#if (defined(_MSVC_LANG) && (_MSVC_LANG >= 201402L))
-  #undef  ARMA_USE_CXX11
-  #define ARMA_USE_CXX11
+#undef ENS_HAVE_CXX14
+
+#if (__cplusplus >= 201402L)
+  #define ENS_HAVE_CXX14
+#endif
+
+#if defined(_MSVC_LANG)
+  #if (_MSVC_LANG >= 201402L)
+    #undef  ENS_HAVE_CXX14
+    #define ENS_HAVE_CXX14
+  #endif
+#endif
+
+#if !defined(ENS_HAVE_CXX14)
+  #error "*** C++14 compiler required; enable C++14 mode in your compiler, or use an earlier version of ensmallen"
 #endif

 #include <armadillo>

-#if !defined(ARMA_USE_CXX11)
-  // armadillo automatically enables ARMA_USE_CXX11
-  // when a C++11/C++14/C++17/etc compiler is detected
-  #error "please enable C++11/C++14 mode in your compiler"
-#endif
-
-#if ((ARMA_VERSION_MAJOR < 8) || ((ARMA_VERSION_MAJOR == 8) && (ARMA_VERSION_MINOR < 400)))
-  #error "need Armadillo version 8.400 or later"
+#if ((ARMA_VERSION_MAJOR < 10) || ((ARMA_VERSION_MAJOR == 10) && (ARMA_VERSION_MINOR < 8)))
+  #error "need Armadillo version 10.8 or newer"
 #endif

 #include <cctype>
@@ -65,6 +70,9 @@

 #include "ensmallen_bits/utility/any.hpp"
 #include "ensmallen_bits/utility/arma_traits.hpp"
+#include "ensmallen_bits/utility/indicators/epsilon.hpp"
+#include "ensmallen_bits/utility/indicators/igd.hpp"
+#include "ensmallen_bits/utility/indicators/igd_plus.hpp"

 // Contains traits, must be placed before report callback.
 #include "ensmallen_bits/function.hpp" // TODO: should move to function/
@@ -72,22 +80,32 @@
 // Callbacks.
 #include "ensmallen_bits/callbacks/callbacks.hpp"
 #include "ensmallen_bits/callbacks/early_stop_at_min_loss.hpp"
+#include "ensmallen_bits/callbacks/grad_clip_by_norm.hpp"
+#include "ensmallen_bits/callbacks/grad_clip_by_value.hpp"
 #include "ensmallen_bits/callbacks/print_loss.hpp"
 #include "ensmallen_bits/callbacks/progress_bar.hpp"
+#include "ensmallen_bits/callbacks/query_front.hpp"
 #include "ensmallen_bits/callbacks/report.hpp"
 #include "ensmallen_bits/callbacks/store_best_coordinates.hpp"
 #include "ensmallen_bits/callbacks/timer_stop.hpp"

 #include "ensmallen_bits/problems/problems.hpp" // TODO: should move to another place

+#include "ensmallen_bits/ada_belief/ada_belief.hpp"
 #include "ensmallen_bits/ada_bound/ada_bound.hpp"
 #include "ensmallen_bits/ada_delta/ada_delta.hpp"
 #include "ensmallen_bits/ada_grad/ada_grad.hpp"
+#include "ensmallen_bits/ada_sqrt/ada_sqrt.hpp"
 #include "ensmallen_bits/adam/adam.hpp"
+#include "ensmallen_bits/demon_adam/demon_adam.hpp"
+#include "ensmallen_bits/demon_sgd/demon_sgd.hpp"
 #include "ensmallen_bits/qhadam/qhadam.hpp"
 #include "ensmallen_bits/aug_lagrangian/aug_lagrangian.hpp"
 #include "ensmallen_bits/bigbatch_sgd/bigbatch_sgd.hpp"
 #include "ensmallen_bits/cmaes/cmaes.hpp"
+#include "ensmallen_bits/cmaes/active_cmaes.hpp"
+#include "ensmallen_bits/cmaes/pop_cmaes.hpp"
+#include "ensmallen_bits/cd/cd.hpp"
 #include "ensmallen_bits/cne/cne.hpp"
 #include "ensmallen_bits/de/de.hpp"
 #include "ensmallen_bits/eve/eve.hpp"
@@ -100,6 +118,8 @@
 #include "ensmallen_bits/katyusha/katyusha.hpp"
 #include "ensmallen_bits/lbfgs/lbfgs.hpp"
 #include "ensmallen_bits/lookahead/lookahead.hpp"
+#include "ensmallen_bits/agemoea/agemoea.hpp"
+#include "ensmallen_bits/moead/moead.hpp"
 #include "ensmallen_bits/nsga2/nsga2.hpp"
 #include "ensmallen_bits/padam/padam.hpp"
 #include "ensmallen_bits/parallel_sgd/parallel_sgd.hpp"
@@ -108,7 +128,6 @@

 #include "ensmallen_bits/sa/sa.hpp"
 #include "ensmallen_bits/sarah/sarah.hpp"
-#include "ensmallen_bits/scd/scd.hpp"
 #include "ensmallen_bits/sdp/sdp.hpp"
 #include "ensmallen_bits/sdp/lrsdp.hpp"
 #include "ensmallen_bits/sdp/primal_dual.hpp"
@@ -125,5 +144,6 @@
 #include "ensmallen_bits/svrg/svrg.hpp"
 #include "ensmallen_bits/swats/swats.hpp"
 #include "ensmallen_bits/wn_grad/wn_grad.hpp"
+#include "ensmallen_bits/yogi/yogi.hpp"

 #endif
@@ -0,0 +1,186 @@
+/**
+ * @file ada_belief.hpp
+ * @author Marcus Edel
+ *
+ * Class wrapper for the AdaBelief update Policy. The intuition for AdaBelief is
+ * to adapt the stepsize according to the "belief" in the current gradient
+ * direction.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_BELIEF_HPP
+#define ENSMALLEN_ADA_BELIEF_HPP
+
+#include <ensmallen_bits/sgd/sgd.hpp>
+#include "ada_belief_update.hpp"
+
+namespace ens {
+
+/**
+ * The intuition for AdaBelief is to adapt the stepsize according to the
+ * "belief" in the current gradient direction. For more information, see the
+ * following.
+ *
+ * @code
+ * @misc{zhuang2020adabelief,
+ *   title         = {AdaBelief Optimizer: Adapting Stepsizes by the Belief in
+ *                    Observed Gradients},
+ *   author        = {Juntang Zhuang and Tommy Tang and Sekhar Tatikonda and
+ *                    Nicha Dvornek and Yifan Ding and Xenophon Papademetris
+ *                    and James S. Duncan},
+ *   year          = {2020},
+ *   eprint        = {2010.07468},
+ *   archivePrefix = {arXiv},
+ * }
+ * @endcode
+ *
+ * AdaBelief can optimize differentiable separable functions. For more details,
+ * see the documentation on function types included with this distribution or
+ * on the ensmallen website.
+ */
+class AdaBelief
+{
+ public:
+  /**
+   * Construct the AdaBelief optimizer with the given function and parameters.
+   * AdaBelief is sensitive to its parameters and hence a good hyperparameter
+   * selection is necessary as its default may not fit every case.
+   *
+   * The maximum number of iterations refers to the maximum number of
+   * points that are processed (i.e., one iteration equals one point; one
+   * iteration does not equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in a single step.
+   * @param beta1 The exponential decay rate for the 1st moment estimates.
+   * @param beta2 The exponential decay rate for the 2nd moment estimates.
+   * @param epsilon A small constant for numerical stability.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *     limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *     function is visited in linear order.
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *     call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
+   */
+  AdaBelief(const double stepSize = 0.001,
+            const size_t batchSize = 32,
+            const double beta1 = 0.9,
+            const double beta2 = 0.999,
+            const double epsilon = 1e-12,
+            const size_t maxIterations = 100000,
+            const double tolerance = 1e-5,
+            const bool shuffle = true,
+            const bool resetPolicy = true,
+            const bool exactObjective = false);
+
+  /**
+   * Optimize the given function using AdaBelief. The given starting point will
+   * be modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
+  {
+    return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the exponential decay rate for the 1st moment estimates.
+  double Beta1() const { return optimizer.UpdatePolicy().Beta1(); }
+  //! Modify the exponential decay rate for the 1st moment estimates.
+  double& Beta1() { return optimizer.UpdatePolicy().Beta1(); }
+
+  //! Get the exponential decay rate for the 2nd moment estimates.
+  double Beta2() const { return optimizer.UpdatePolicy().Beta2(); }
+  //! Get the second moment coefficient.
+  double& Beta2() { return optimizer.UpdatePolicy().Beta2(); }
+
+  //! Get the value for numerical stability.
+  double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
+  //! Modify the value used for numerical stability.
+  double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
+  //! Get whether or not the update policy parameters are reset before
+  //! Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+  private:
+  //! The Stochastic Gradient Descent object with AdaBelief policy.
+  SGD<AdaBeliefUpdate> optimizer;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "ada_belief_impl.hpp"
+
+#endif
@@ -0,0 +1,44 @@
+/**
+ * @file ada_belief_impl.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of AdaBelief class wrapper.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_BELIEF_ADA_BELIEF_IMPL_HPP
+#define ENSMALLEN_ADA_BELIEF_ADA_BELIEF_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "ada_belief.hpp"
+
+namespace ens {
+
+inline AdaBelief::AdaBelief(
+    const double stepSize,
+    const size_t batchSize,
+    const double beta1,
+    const double beta2,
+    const double epsilon,
+    const size_t maxIterations,
+    const double tolerance,
+    const bool shuffle,
+    const bool resetPolicy,
+    const bool exactObjective) :
+    optimizer(stepSize,
+              batchSize,
+              maxIterations,
+              tolerance,
+              shuffle,
+              AdaBeliefUpdate(epsilon, beta1, beta2),
+              NoDecay(),
+              resetPolicy,
+              exactObjective)
+{ /* Nothing to do. */ }
+
+} // namespace ens
+
+ #endif
@@ -0,0 +1,153 @@
+/**
+ * @file ada_belief_update.hpp
+ * @author Marcus Edel
+ *
+ * AdaBelief optimizer update policy. The intuition for AdaBelief is to adapt
+ * the stepsize according to the "belief" in the current gradient direction.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_BELIEF_ADA_BELIEF_UPDATE_HPP
+#define ENSMALLEN_ADA_BELIEF_ADA_BELIEF_UPDATE_HPP
+
+namespace ens {
+
+/**
+ * The intuition for AdaBelief is to adapt the stepsize according to the
+ * "belief" in the current gradient direction.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{zhuang2020adabelief,
+ *   title         = {AdaBelief Optimizer: Adapting Stepsizes by the Belief in
+ *                    Observed Gradients},
+ *   author        = {Juntang Zhuang and Tommy Tang and Sekhar Tatikonda and
+ *                    Nicha Dvornek and Yifan Ding and Xenophon Papademetris
+ *                    and James S. Duncan},
+ *   year          = {2020},
+ *   eprint        = {2010.07468},
+ *   archivePrefix = {arXiv},
+ * }
+ * @endcode
+ */
+class AdaBeliefUpdate
+{
+ public:
+  /**
+   * Construct the AdaBelief update policy with the given parameters.
+   *
+   * @param epsilon A small constant for numerical stability.
+   * @param beta1 The exponential decay rate for the 1st moment estimates.
+   * @param beta2 The exponential decay rate for the 2nd moment estimates.
+   */
+  AdaBeliefUpdate(const double epsilon = 1e-8,
+                  const double beta1 = 0.9,
+                  const double beta2 = 0.999) :
+    epsilon(epsilon),
+    beta1(beta1),
+    beta2(beta2)
+  {
+    // Nothing to do.
+  }
+
+  //! Get the value for numerical stability.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used for numerical stability.
+  double& Epsilon() { return epsilon; }
+
+  //! Get the exponential decay rate for the 1st moment estimates.
+  double Beta1() const { return beta1; }
+  //! Modify the exponential decay rate for the 1st moment estimates.
+  double& Beta1() { return beta1; }
+
+  //! Get the exponential decay rate for the 2nd moment estimates.
+  double Beta2() const { return beta2; }
+  //! Modify the exponential decay rate for the 2nd moment estimates.
+  double& Beta2() { return beta2; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent AdaBeliefUpdate object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AdaBeliefUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent),
+        iteration(0)
+    {
+      m.zeros(rows, cols);
+      s.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for AdaBelief.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Increment the iteration counter variable.
+      ++iteration;
+
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      s *= parent.beta2;
+      s += (1 - parent.beta2) * arma::pow(gradient - m, 2.0) + parent.epsilon;
+
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
+
+      // And update the iterate.
+      iterate -= ((m / biasCorrection1) * stepSize) / (arma::sqrt(s /
+          biasCorrection2) + parent.epsilon);
+    }
+
+   private:
+    //! Instantiated parent object.
+    AdaBeliefUpdate& parent;
+
+    //! The exponential moving average of gradient values.
+    GradType m;
+
+    // The exponential moving average of squared gradient values.
+    GradType s;
+
+    // The number of iterations.
+    size_t iteration;
+  };
+
+ private:
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+
+  // The xponential decay rate for the 1st moment estimates.
+  double beta1;
+
+  // The exponential decay rate for the 2nd moment estimates.
+  double beta2;
+};
+
+} // namespace ens
+
+#endif
@@ -56,8 +56,7 @@ class AdaBoundUpdate
    gamma(gamma),
    epsilon(epsilon),
    beta1(beta1),
-    beta2(beta2),
-    iteration(0)
+    beta2(beta2)
  {
    // Nothing to do.
  }
@@ -87,11 +86,6 @@ class AdaBoundUpdate
  //! Modify the second moment coefficient.
  double& Beta2() { return beta2; }

-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
  /**
   * The UpdatePolicyType policy classes must contain an internal 'Policy'
   * template class with two template arguments: MatType and GradType.  This is
@@ -111,7 +105,7 @@ class AdaBoundUpdate
     * @param cols Number of columns in the gradient matrix.
     */
    Policy(AdaBoundUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent), first(true), initialStepSize(0)
+        parent(parent), first(true), initialStepSize(0), iteration(0)
    {
      m.zeros(rows, cols);
      v.zeros(rows, cols);
@@ -139,7 +133,7 @@ class AdaBoundUpdate
      }

      // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;

      // Decay the first and second moment running average coefficient.
      m *= parent.beta1;
@@ -148,16 +142,12 @@ class AdaBoundUpdate
      v *= parent.beta2;
      v += (1 - parent.beta2) * (gradient % gradient);

-      const ElemType biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
-      const ElemType biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
+      const ElemType biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
+      const ElemType biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);

      const ElemType fl = parent.finalLr * stepSize / initialStepSize;
-      const ElemType lower = fl * (1.0 - 1.0 / (parent.gamma *
-          parent.iteration + 1));
-      const ElemType upper = fl * (1.0 + 1.0 / (parent.gamma *
-          parent.iteration));
+      const ElemType lower = fl * (1.0 - 1.0 / (parent.gamma * iteration + 1));
+      const ElemType upper = fl * (1.0 + 1.0 / (parent.gamma * iteration));

       // Applies bounds on actual learning rate.
      iterate -= arma::clamp((stepSize *
@@ -180,6 +170,9 @@ class AdaBoundUpdate

    // The initial (Adam) learning rate.
    double initialStepSize;
+
+    // The number of iterations.
+    size_t iteration;
  };

 private:
@@ -197,9 +190,6 @@ class AdaBoundUpdate

  // The second moment coefficient.
  double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };

 } // namespace ens
@@ -56,8 +56,7 @@ class AMSBoundUpdate
      gamma(gamma),
      epsilon(epsilon),
      beta1(beta1),
-      beta2(beta2),
-      iteration(0)
+      beta2(beta2)
  {
    // Nothing to do.
  }
@@ -87,11 +86,6 @@ class AMSBoundUpdate
  //! Modify the second moment coefficient.
  double& Beta2() { return beta2; }

-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
  /**
   * The UpdatePolicyType policy classes must contain an internal 'Policy'
   * template class with two template arguments: MatType and GradType.  This is
@@ -111,7 +105,7 @@ class AMSBoundUpdate
     * @param cols Number of columns in the gradient matrix.
     */
    Policy(AMSBoundUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent), first(true), initialStepSize(0)
+        parent(parent), first(true), initialStepSize(0), iteration(0)
    {
      m.zeros(rows, cols);
      v.zeros(rows, cols);
@@ -140,7 +134,7 @@ class AMSBoundUpdate
      }

      // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;

      // Decay the first and second moment running average coefficient.
      m *= parent.beta1;
@@ -149,16 +143,12 @@ class AMSBoundUpdate
      v *= parent.beta2;
      v += (1 - parent.beta2) * (gradient % gradient);

-      const ElemType biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
-      const ElemType biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
+      const ElemType biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
+      const ElemType biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);

      const ElemType fl = parent.finalLr * stepSize / initialStepSize;
-      const ElemType lower = fl * (1.0 - 1.0 / (parent.gamma *
-          parent.iteration + 1));
-      const ElemType upper = fl * (1.0 + 1.0 / (parent.gamma *
-          parent.iteration));
+      const ElemType lower = fl * (1.0 - 1.0 / (parent.gamma * iteration + 1));
+      const ElemType upper = fl * (1.0 + 1.0 / (parent.gamma * iteration));

      // Element wise maximum of past and present squared gradients.
      vImproved = arma::max(vImproved, v);
@@ -187,6 +177,9 @@ class AMSBoundUpdate

    // The optimal squared gradient value.
    GradType vImproved;
+
+    // The number of iterations.
+    size_t iteration;
  };

 private:
@@ -204,9 +197,6 @@ class AMSBoundUpdate

  // The second moment coefficient.
  double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };

 } // namespace ens
@@ -0,0 +1,168 @@
+/**
+ * @file ada_sqrt.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of the AdaSqrt optimizer. AdaSqrt is an optimizer that
+ * chooses learning rate dynamically by adapting to the data and iteration.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * terms of the 3-clause BSD license. You should have received a copy of the
+ * 3-clause BSD license along with ensmallen. If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_SQRT_ADA_SQRT_HPP
+#define ENSMALLEN_ADA_SQRT_ADA_SQRT_HPP
+
+#include "../sgd/sgd.hpp"
+#include "ada_sqrt_update.hpp"
+
+namespace ens {
+
+/**
+ * AdaSqrt is a modified version of stochastic gradient descent which performs
+ * larger updates for more sparse parameters and smaller updates for less sparse
+ * parameters.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{hu2019secondorder,
+ *   title  = {Second-order Information in First-order Optimization Methods},
+ *   author = {Yuzheng Hu and Licong Lin and Shange Tang},
+ *   year   = {2019},
+ *   eprint = {1912.09926},
+ * }
+ * @endcode
+ *
+ * AdaSqrt can optimize differentiable separable functions.  For more details,
+ * see the documentation on function types included with this distribution or on
+ * the ensmallen website.
+ */
+class AdaSqrt
+{
+ public:
+  /**
+   * Construct the AdaSqrt optimizer with the given function and parameters.
+   * The defaults here are not necessarily good for the given problem, so it is
+   * suggested that the values used be tailored to the task at hand. The
+   * maximum number of iterations refers to the maximum number of points that
+   * are processed (i.e., one iteration equals one point; one iteration does not
+   * equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in one step.
+   * @param epsilon Value used to initialise the squared gradient parameter.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *        limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *        function is visited in linear order.
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *        call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
+   */
+  AdaSqrt(const double stepSize = 0.01,
+          const size_t batchSize = 32,
+          const double epsilon = 1e-8,
+          const size_t maxIterations = 100000,
+          const double tolerance = 1e-5,
+          const bool shuffle = true,
+          const bool resetPolicy = true,
+          const bool exactObjective = false);
+
+  /**
+   * Optimize the given function using AdaSqrt. The given starting point will
+   * be modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
+  {
+    return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
+  //! Get whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+ private:
+  //! The Stochastic Gradient Descent object with AdaSqrt policy.
+  SGD<AdaSqrtUpdate> optimizer;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "ada_sqrt_impl.hpp"
+
+#endif
@@ -0,0 +1,38 @@
+/**
+ * @file ada_sqrt_impl.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of AdaSqrt optimizer.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_SQRT_ADA_SQRT_IMPL_HPP
+#define ENSMALLEN_ADA_SQRT_ADA_SQRT_IMPL_HPP
+
+namespace ens {
+
+inline AdaSqrt::AdaSqrt(const double stepSize,
+                        const size_t batchSize,
+                        const double epsilon,
+                        const size_t maxIterations,
+                        const double tolerance,
+                        const bool shuffle,
+                        const bool resetPolicy,
+                        const bool exactObjective) :
+    optimizer(stepSize,
+              batchSize,
+              maxIterations,
+              tolerance,
+              shuffle,
+              AdaSqrtUpdate(epsilon),
+              NoDecay(),
+              resetPolicy,
+              exactObjective)
+{ /* Nothing to do. */ }
+
+} // namespace ens
+
+#endif
@@ -0,0 +1,118 @@
+/**
+ * @file ada_sqrt_update.hpp
+ * @author Marcus Edel
+ *
+ * AdaSqrt update for Stochastic Gradient Descent.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_SQRT_ADA_SQRT_UPDATE_HPP
+#define ENSMALLEN_ADA_SQRT_ADA_SQRT_UPDATE_HPP
+
+namespace ens {
+
+/**
+ * Implementation of the AdaSqrt update policy. AdaSqrt update policy chooses
+ * learning rate dynamically by adapting to the data and iteration.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{hu2019secondorder,
+ *   title  = {Second-order Information in First-order Optimization Methods},
+ *   author = {Yuzheng Hu and Licong Lin and Shange Tang},
+ *   year   = {2019},
+ *   eprint = {1912.09926},
+ * }
+ * @endcode
+ *
+ */
+class AdaSqrtUpdate
+{
+ public:
+  /**
+   * Construct the AdaSqrt update policy with given epsilon parameter.
+   *
+   * @param epsilon The epsilon value used to initialise the squared gradient
+   *        parameter.
+   */
+  AdaSqrtUpdate(const double epsilon = 1e-8) : epsilon(epsilon)
+  {
+    // Nothing to do.
+  }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return epsilon; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD optimizer before the start of the
+     * iteration update process. In AdaSqrt update policy, squared gradient
+     * matrix is initialized to the zeros matrix with the same size as gradient
+     * matrix (see ens::SGD<>).
+     *
+     * @param parent Instantiated parent class.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AdaSqrtUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent),
+        squaredGradient(rows, cols),
+        iteration(0)
+    {
+      // Initialize an empty matrix for sum of squares of parameter gradient.
+      squaredGradient.zeros();
+    }
+
+    /**
+     * Update step for SGD. The AdaSqrt update adapts the learning rate by
+     * performing larger updates for more sparse parameters and smaller updates
+     * for less sparse parameters.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      ++iteration;
+
+      squaredGradient += arma::square(gradient);
+
+      iterate -= stepSize * std::sqrt(iteration) * gradient /
+          (squaredGradient + parent.epsilon);
+    }
+
+   private:
+    // Instantiated parent class.
+    AdaSqrtUpdate& parent;
+    // The squared gradient matrix.
+    GradType squaredGradient;
+    // The number of iterations.
+    size_t iteration;
+  };
+
+ private:
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+};
+
+} // namespace ens
+
+#endif
@@ -52,8 +52,7 @@ class AdamUpdate
             const double beta2 = 0.999) :
    epsilon(epsilon),
    beta1(beta1),
-    beta2(beta2),
-    iteration(0)
+    beta2(beta2)
  {
    // Nothing to do.
  }
@@ -73,11 +72,6 @@ class AdamUpdate
  //! Modify the second moment coefficient.
  double& Beta2() { return beta2; }

-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
  /**
   * The UpdatePolicyType policy classes must contain an internal 'Policy'
   * template class with two template arguments: MatType and GradType.  This is
@@ -97,7 +91,8 @@ class AdamUpdate
     * @param cols Number of columns in the gradient matrix.
     */
    Policy(AdamUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        iteration(0)
    {
      m.zeros(rows, cols);
      v.zeros(rows, cols);
@@ -115,7 +110,7 @@ class AdamUpdate
                const GradType& gradient)
    {
      // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;

      // And update the iterate.
      m *= parent.beta1;
@@ -124,10 +119,8 @@ class AdamUpdate
      v *= parent.beta2;
      v += (1 - parent.beta2) * (gradient % gradient);

-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);

      /**
       * It should be noted that the term, m / (arma::sqrt(v) + eps), in the
@@ -147,6 +140,9 @@ class AdamUpdate

    // The exponential moving average of squared gradient values.
    GradType v;
+
+    // The number of iterations.
+    size_t iteration;
  };

 private:
@@ -158,9 +154,6 @@ class AdamUpdate

  // The second moment coefficient.
  double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };

 } // namespace ens
@@ -54,8 +54,7 @@ class AdaMaxUpdate
               const double beta2 = 0.999) :
    epsilon(epsilon),
    beta1(beta1),
-    beta2(beta2),
-    iteration(0)
+    beta2(beta2)
  {
    // Nothing to do.
  }
@@ -75,11 +74,6 @@ class AdaMaxUpdate
  //! Modify the second moment coefficient.
  double& Beta2() { return beta2; }

-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
  /**
   * The UpdatePolicyType policy classes must contain an internal 'Policy'
   * template class with two template arguments: MatType and GradType.  This is
@@ -99,7 +93,8 @@ class AdaMaxUpdate
     * @param cols Number of columns in the gradient matrix.
     */
    Policy(AdaMaxUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        iteration(0)
    {
      m.zeros(rows, cols);
      u.zeros(rows, cols);
@@ -117,7 +112,7 @@ class AdaMaxUpdate
                const GradType& gradient)
    {
      // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;

      // And update the iterate.
      m *= parent.beta1;
@@ -127,8 +122,7 @@ class AdaMaxUpdate
      u *= parent.beta2;
      u = arma::max(u, arma::abs(gradient));

-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);

      if (biasCorrection1 != 0)
        iterate -= (stepSize / biasCorrection1 * m / (u + parent.epsilon));
@@ -141,6 +135,8 @@ class AdaMaxUpdate
    GradType m;
    // The exponentially weighted infinity norm.
    GradType u;
+    // The number of iterations.
+    size_t iteration;
  };

 private:
@@ -152,9 +148,6 @@ class AdaMaxUpdate

  // The second moment coefficient.
  double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };

 } // namespace ens
@@ -47,8 +47,7 @@ class AMSGradUpdate
                const double beta2 = 0.999) :
    epsilon(epsilon),
    beta1(beta1),
-    beta2(beta2),
-    iteration(0)
+    beta2(beta2)
  {
    // Nothing to do.
  }
@@ -68,11 +67,6 @@ class AMSGradUpdate
  //! Modify the second moment coefficient.
  double& Beta2() { return beta2; }

-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
  /**
   * The UpdatePolicyType policy classes must contain an internal 'Policy'
   * template class with two template arguments: MatType and GradType.  This is
@@ -92,7 +86,8 @@ class AMSGradUpdate
     * @param cols Number of columns in the gradient matrix.
     */
    Policy(AMSGradUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        iteration(0)
    {
      m.zeros(rows, cols);
      v.zeros(rows, cols);
@@ -111,7 +106,7 @@ class AMSGradUpdate
                const GradType& gradient)
    {
      // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;

      // And update the iterate.
      m *= parent.beta1;
@@ -120,10 +115,8 @@ class AMSGradUpdate
      v *= parent.beta2;
      v += (1 - parent.beta2) * (gradient % gradient);

-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);

      // Element wise maximum of past and present squared gradients.
      vImproved = arma::max(vImproved, v);
@@ -144,6 +137,9 @@ class AMSGradUpdate

    // The optimal squared gradient value.
    GradType vImproved;
+
+    // The number of iterations.
+    size_t iteration;
  };

 private:
@@ -155,9 +151,6 @@ class AMSGradUpdate

  // The second moment coefficient.
  double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };

 } // namespace ens
@@ -50,8 +50,7 @@ class NadamUpdate
      epsilon(epsilon),
      beta1(beta1),
      beta2(beta2),
-      scheduleDecay(scheduleDecay),
-      iteration(0)
+      scheduleDecay(scheduleDecay)
  {
    // Nothing to do.
  }
@@ -76,11 +75,6 @@ class NadamUpdate
  //! Modify the decay parameter for decay coefficients
  double& ScheduleDecay() { return scheduleDecay; }

-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
  /**
   * The UpdatePolicyType policy classes must contain an internal 'Policy'
   * template class with two template arguments: MatType and GradType.  This is
@@ -101,7 +95,8 @@ class NadamUpdate
     */
    Policy(NadamUpdate& parent, const size_t rows, const size_t cols) :
        parent(parent),
-        cumBeta1(1)
+        cumBeta1(1),
+        iteration(0)
    {
      m.zeros(rows, cols);
      v.zeros(rows, cols);
@@ -119,7 +114,7 @@ class NadamUpdate
                const GradType& gradient)
    {
      // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;

      // And update the iterate.
      m *= parent.beta1;
@@ -129,18 +124,15 @@ class NadamUpdate
      v += (1 - parent.beta2) * gradient % gradient;

      double beta1T = parent.beta1 * (1 - (0.5 *
-          std::pow(0.96, parent.iteration * parent.scheduleDecay)));
+          std::pow(0.96, iteration * parent.scheduleDecay)));

      double beta1T1 = parent.beta1 * (1 - (0.5 *
-          std::pow(0.96, (parent.iteration + 1) * parent.scheduleDecay)));
+          std::pow(0.96, (iteration + 1) * parent.scheduleDecay)));

      cumBeta1 *= beta1T;

      const double biasCorrection1 = 1.0 - cumBeta1;
-
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
-
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
      const double biasCorrection3 = 1.0 - (cumBeta1 * beta1T1);

      /* Note :- arma::sqrt(v) + epsilon * sqrt(biasCorrection2) is approximated
@@ -163,6 +155,9 @@ class NadamUpdate

    // The cumulative product of decay coefficients.
    double cumBeta1;
+
+    // The number of iterations.
+    size_t iteration;
  };

 private:
@@ -177,9 +172,6 @@ class NadamUpdate

  // The decay parameter for decay coefficients.
  double scheduleDecay;
-
-  // The number of iterations.
-  size_t iteration;
 };

 } // namespace ens
@@ -50,8 +50,7 @@ class NadaMaxUpdate
      epsilon(epsilon),
      beta1(beta1),
      beta2(beta2),
-      scheduleDecay(scheduleDecay),
-      iteration(0)
+      scheduleDecay(scheduleDecay)
  {
    // Nothing to do.
  }
@@ -76,11 +75,6 @@ class NadaMaxUpdate
  //! Modify the decay parameter for decay coefficients
  double& ScheduleDecay() { return scheduleDecay; }

-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
  /**
   * The UpdatePolicyType policy classes must contain an internal 'Policy'
   * template class with two template arguments: MatType and GradType.  This is
@@ -101,7 +95,8 @@ class NadaMaxUpdate
     */
    Policy(NadaMaxUpdate& parent, const size_t rows, const size_t cols) :
        parent(parent),
-        cumBeta1(1)
+        cumBeta1(1),
+        iteration(0)
    {
      m.zeros(rows, cols);
      u.zeros(rows, cols);
@@ -119,7 +114,7 @@ class NadaMaxUpdate
                const GradType& gradient)
    {
      // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;

      // And update the iterate.
      m *= parent.beta1;
@@ -128,10 +123,10 @@ class NadaMaxUpdate
      u = arma::max(u * parent.beta2, arma::abs(gradient));

      double beta1T = parent.beta1 * (1 - (0.5 *
-          std::pow(0.96, parent.iteration * parent.scheduleDecay)));
+          std::pow(0.96, iteration * parent.scheduleDecay)));

      double beta1T1 = parent.beta1 * (1 - (0.5 *
-          std::pow(0.96, (parent.iteration + 1) * parent.scheduleDecay)));
+          std::pow(0.96, (iteration + 1) * parent.scheduleDecay)));

      cumBeta1 *= beta1T;

@@ -158,6 +153,9 @@ class NadaMaxUpdate

    // The cumulative product of decay coefficients.
    double cumBeta1;
+
+    // The number of iterations.
+    size_t iteration;
  };

 private:
@@ -172,9 +170,6 @@ class NadaMaxUpdate

  // The decay parameter for decay coefficients.
  double scheduleDecay;
-
-  // The number of iterations.
-  size_t iteration;
 };

 } // namespace ens
@@ -51,8 +51,7 @@ class OptimisticAdamUpdate
                       const double beta2 = 0.999) :
    epsilon(epsilon),
    beta1(beta1),
-    beta2(beta2),
-    iteration(0)
+    beta2(beta2)
  {
    // Nothing to do.
  }
@@ -72,11 +71,6 @@ class OptimisticAdamUpdate
  //! Modify the second moment coefficient.
  double& Beta2() { return beta2; }

-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
  /**
   * The UpdatePolicyType policy classes must contain an internal 'Policy'
   * template class with two template arguments: MatType and GradType.  This is
@@ -96,7 +90,8 @@ class OptimisticAdamUpdate
     * @param cols Number of columns in the gradient matrix.
     */
    Policy(OptimisticAdamUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        iteration(0)
    {
      m.zeros(rows, cols);
      v.zeros(rows, cols);
@@ -115,7 +110,7 @@ class OptimisticAdamUpdate
                const GradType& gradient)
    {
      // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;

      // And update the iterate.
      m *= parent.beta1;
@@ -124,13 +119,10 @@ class OptimisticAdamUpdate
      v *= parent.beta2;
      v += (1 - parent.beta2) * arma::square(gradient);

-      GradType mCorrected = m / (1.0 - std::pow(parent.beta1,
-          parent.iteration));
-      GradType vCorrected = v / (1.0 - std::pow(parent.beta2,
-          parent.iteration));
+      GradType mCorrected = m / (1.0 - std::pow(parent.beta1, iteration));
+      GradType vCorrected = v / (1.0 - std::pow(parent.beta2, iteration));

-      GradType update = mCorrected /
-          (arma::sqrt(vCorrected) + parent.epsilon);
+      GradType update = mCorrected / (arma::sqrt(vCorrected) + parent.epsilon);

      iterate -= (2 * stepSize * update - stepSize * g);

@@ -149,6 +141,9 @@ class OptimisticAdamUpdate

    // The previous update.
    GradType g;
+
+    // The number of iterations.
+    size_t iteration;
  };

 private:
@@ -160,9 +155,6 @@ class OptimisticAdamUpdate

  // The second moment coefficient.
  double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };

 } // namespace ens
@@ -0,0 +1,496 @@
+/**
+ * @file agemoea.hpp
+ * @author Satyam Shukla
+ *
+ * AGE-MOEA is a multi-objective optimization algorithm, widely used in
+ * many real-world applications. AGE-MOEA generates offsprings using
+ * crossover and mutation and then selects the next generation according
+ * to non-dominated-sorting and survival score comparison.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_AGEMOEA_AGEMOEA_HPP
+#define ENSMALLEN_AGEMOEA_AGEMOEA_HPP
+
+namespace ens {
+
+/**
+ * This class implements the AGEMOEA algorithm.
+ *
+ * The algorithm works by generating a candidate population from a fixed
+ * starting point. At each stage of optimization, a new population of children
+ * is generated. This new population along with its predecessor is sorted using
+ * non-domination as the metric. Following this, the population is further
+ * segregated in fronts. A new population is generated from these fronts having
+ * size equal to that of the starting population.
+ *
+ * During evolution, two parents are randomly chosen using binary tournament
+ * selection. A pair of children are generated by crossing over these two
+ * candidates followed by mutation.
+ *
+ * The best front (Pareto optimal) is returned by the Optimize() method.
+ *
+ * For more information, see the following:
+ *
+ * @code
+ * @inproceedings{panichella2019adaptive,
+ * title={An adaptive evolutionary algorithm based on non-euclidean geometry for many-objective optimization},
+ * author={Panichella, Annibale},
+ * booktitle={Proceedings of the genetic and evolutionary computation conference},
+ * pages={595--603},
+ * year={2019}
+ * }
+ * @endcode
+ *
+ */
+class AGEMOEA
+{
+ public:
+  /**
+   * Constructor for the AGE-MOEA optimizer.
+   *
+   * The default values provided over here are not necessarily suitable for a
+   * given function. Therefore it is highly recommended to adjust the
+   * parameters according to the problem.
+   *
+   * @param populationSize The number of candidates in the population.
+   *     This should be atleast 4 in size and a multiple of 4.
+   * @param maxGenerations The maximum number of generations allowed for NSGA-II.
+   * @param crossoverProb The probability that a crossover will occur.
+   * @param distributionIndex The crowding degree of the mutation.
+   * @param epsilon The minimum difference required to distinguish between
+   *     candidate solutions.
+   * @param eta The distance parameters of the crossover distribution.
+   * @param lowerBound Lower bound of the coordinates of the initial population.
+   * @param upperBound Upper bound of the coordinates of the initial population.
+   */
+  AGEMOEA(const size_t populationSize = 100,
+          const size_t maxGenerations = 2000,
+          const double crossoverProb = 0.6,
+          const double distributionIndex = 20,
+          const double epsilon = 1e-6,
+          const double eta = 20,
+          const arma::vec& lowerBound = arma::zeros(1, 1),
+          const arma::vec& upperBound = arma::ones(1, 1));
+
+  /**
+   * Constructor for the AGE-MOEA optimizer. This constructor provides an overload
+   * to use `lowerBound` and `upperBound` of type double.
+   *
+   * The default values provided over here are not necessarily suitable for a
+   * given function. Therefore it is highly recommended to adjust the
+   * parameters according to the problem.
+   *
+   * @param populationSize The number of candidates in the population.
+   *     This should be atleast 4 in size and a multiple of 4.
+   * @param maxGenerations The maximum number of generations allowed for NSGA-II.
+   * @param crossoverProb The probability that a crossover will occur.
+   * @param distributionIndex The crowding degree of the mutation.
+   * @param epsilon The minimum difference required to distinguish between
+   *     candidate solutions.
+   * @param eta The distance parameters of the crossover distribution
+   * @param lowerBound Lower bound of the coordinates of the initial population.
+   * @param upperBound Upper bound of the coordinates of the initial population.
+   */
+  AGEMOEA(const size_t populationSize = 100,
+          const size_t maxGenerations = 2000,
+          const double crossoverProb = 0.6,
+          const double distributionIndex = 20,
+          const double epsilon = 1e-6,
+          const double eta = 20,
+          const double lowerBound = 0,
+          const double upperBound = 1);
+
+  /**
+   * Optimize a set of objectives. The initial population is generated using the
+   * starting point. The output is the best generated front.
+   *
+   * @tparam ArbitraryFunctionType std::tuple of multiple objectives.
+   * @tparam MatType Type of matrix to optimize.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param objectives Vector of objective functions to optimize for.
+   * @param iterate Starting point.
+   * @param callbacks Callback functions.
+   * @return MatType::elem_type The minimum of the accumulated sum over the
+   *     objective values in the best front.
+   */
+  template<typename MatType,
+           typename... ArbitraryFunctionType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(
+      std::tuple<ArbitraryFunctionType...>& objectives,
+      MatType& iterate,
+      CallbackTypes&&... callbacks);
+
+  //! Get the population size.
+  size_t PopulationSize() const { return populationSize; }
+  //! Modify the population size.
+  size_t& PopulationSize() { return populationSize; }
+
+  //! Get the maximum number of generations.
+  size_t MaxGenerations() const { return maxGenerations; }
+  //! Modify the maximum number of generations.
+  size_t& MaxGenerations() { return maxGenerations; }
+
+  //! Get the crossover rate.
+  double CrossoverRate() const { return crossoverProb; }
+  //! Modify the crossover rate.
+  double& CrossoverRate() { return crossoverProb; }
+
+  //! Retrieve value of the distribution index.
+  double DistributionIndex() const { return distributionIndex; }
+  //! Modify the value of the distribution index.
+  double& DistributionIndex() { return distributionIndex; }
+
+  //! Retrieve value of eta.
+  double Eta() const { return eta; }
+  //! Modify the value of eta.
+  double& Eta() { return eta; }
+
+  //! Get the tolerance.
+  double Epsilon() const { return epsilon; }
+  //! Modify the tolerance.
+  double& Epsilon() { return epsilon; }
+
+  //! Retrieve value of lowerBound.
+  const arma::vec& LowerBound() const { return lowerBound; }
+  //! Modify value of lowerBound.
+  arma::vec& LowerBound() { return lowerBound; }
+
+  //! Retrieve value of upperBound.
+  const arma::vec& UpperBound() const { return upperBound; }
+  //! Modify value of upperBound.
+  arma::vec& UpperBound() { return upperBound; }
+
+  //! Retrieve the Pareto optimal points in variable space. This returns an empty cube
+  //! until `Optimize()` has been called.
+  const arma::cube& ParetoSet() const { return paretoSet; }
+
+  //! Retrieve the best front (the Pareto frontier). This returns an empty cube until
+  //! `Optimize()` has been called.
+  const arma::cube& ParetoFront() const { return paretoFront; }
+
+  /**
+   * Retrieve the best front (the Pareto frontier).  This returns an empty
+   * vector until `Optimize()` has been called.  Note that this function is
+   * deprecated and will be removed in ensmallen 3.x!  Use `ParetoFront()`
+   * instead.
+   */
+  const std::vector<arma::mat>& Front()
+  {
+    if (rcFront.size() == 0)
+    {
+      // Match the old return format.
+      for (size_t i = 0; i < paretoFront.n_slices; ++i)
+      {
+        rcFront.push_back(arma::mat(paretoFront.slice(i)));
+      }
+    }
+
+    return rcFront;
+  }
+
+ private:
+  /**
+   * Evaluate objectives for the elite population.
+   *
+   * @tparam ArbitraryFunctionType std::tuple of multiple function types.
+   * @tparam MatType Type of matrix to optimize.
+   * @param population The elite population.
+   * @param objectives The set of objectives.
+   * @param calculatedObjectives Vector to store calculated objectives.
+   */
+  template<std::size_t I = 0,
+           typename MatType,
+           typename ...ArbitraryFunctionType>
+  typename std::enable_if<I == sizeof...(ArbitraryFunctionType), void>::type
+  EvaluateObjectives(std::vector<MatType>&,
+                     std::tuple<ArbitraryFunctionType...>&,
+                     std::vector<arma::Col<typename MatType::elem_type> >&);
+
+  template<std::size_t I = 0,
+           typename MatType,
+           typename ...ArbitraryFunctionType>
+  typename std::enable_if<I < sizeof...(ArbitraryFunctionType), void>::type
+  EvaluateObjectives(std::vector<MatType>& population,
+                     std::tuple<ArbitraryFunctionType...>& objectives,
+                     std::vector<arma::Col<typename MatType::elem_type> >&
+                     calculatedObjectives);
+
+  /**
+   * Reproduce candidates from the elite population to generate a new
+   * population.
+   *
+   * @tparam MatType Type of matrix to optimize.
+   * @param objectives The set of objectives.
+   * @param lowerBound Lower bound of the coordinates of the initial population.
+   * @param upperBound Upper bound of the coordinates of the initial population.
+   */
+  template<typename MatType>
+  void BinaryTournamentSelection(std::vector<MatType>& population,
+                                 const MatType& lowerBound,
+                                 const MatType& upperBound);
+
+  /**
+   * Crossover two parents to create a pair of new children.
+   *
+   * @tparam MatType Type of matrix to optimize.
+   * @param childA A newly generated candidate.
+   * @param childB Another newly generated candidate.
+   * @param parentA First parent from elite population.
+   * @param parentB Second parent from elite population.
+   * @param lowerBound The lower bound of the objectives.
+   * @param upperBound The upper bound of the objectives.
+   */
+  template<typename MatType>
+  void Crossover(MatType& childA,
+                 MatType& childB,
+                 const MatType& parentA,
+                 const MatType& parentB,
+                 const MatType& lowerBound,
+                 const MatType& upperBound);
+
+  /**
+   * Mutate the coordinates for a candidate.
+   *
+   * @tparam MatType Type of matrix to optimize.
+   * @param candidate The candidate whose coordinates are being modified.
+   * @param mutationRate The probablity of a mutation to occur.
+   * @param lowerBound Lower bound of the coordinates of the initial population.
+   * @param upperBound Upper bound of the coordinates of the initial population.
+   */
+  template<typename MatType>
+  void Mutate(MatType& candidate,
+              double mutationRate,
+              const MatType& lowerBound,
+              const MatType& upperBound);
+
+  /**
+   * Sort the candidate population using their domination count and the set of
+   * dominated nodes.
+   *
+   * @tparam MatType Type of matrix to optimize.
+   * @param fronts The population is sorted into these Pareto fronts. The first
+   *     front is the best, the second worse and so on.
+   * @param ranks The assigned ranks, used for crowding distance based sorting.
+   * @param calculatedObjectives The previously calculated objectives.
+   */
+  template<typename MatType>
+  void FastNonDominatedSort(
+      std::vector<std::vector<size_t> >& fronts,
+      std::vector<size_t>& ranks,
+      std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives);
+
+  /**
+   * Operator to check if one candidate Pareto-dominates the other.
+   *
+   * A candidate is said to dominate the other if it is at least as good as the
+   * other candidate for all the objectives and there exists at least one
+   * objective for which it is strictly better than the other candidate.
+   *
+   * @tparam MatType Type of matrix to optimize.
+   * @param calculatedObjectives The previously calculated objectives.
+   * @param candidateP The candidate being compared from the elite population.
+   * @param candidateQ The candidate being compared against.
+   * @return true if candidateP Pareto dominates candidateQ, otherwise, false.
+   */
+  template<typename MatType>
+  bool Dominates(
+      std::vector<arma::Col<typename MatType::elem_type>>& calculatedObjectives,
+      size_t candidateP,
+      size_t candidateQ);
+
+ /**
+  * Assigns Survival Score metric for sorting.
+  *
+  * @param front The previously generated Pareto fronts.
+  * @param idealPoint The ideal point of teh first front.
+  * @param calculatedObjectives The previously calculated objectives.
+  * @param survivalScore The Survival Score vector to be updated for each individual in the population.
+  * @param normalize The normlization vector of the fronts.
+  * @param dimension The dimension of the first front.
+  * @param fNum teh current front index.
+  */
+  template <typename MatType>
+  void SurvivalScoreAssignment(
+      const std::vector<size_t>& front,
+      const arma::Col<typename MatType::elem_type>& idealPoint,
+      std::vector<arma::Col<typename MatType::elem_type>>& calculatedObjectives,
+      std::vector<typename MatType::elem_type>& survivalScore,
+      arma::Col<typename MatType::elem_type>& normalize,
+      double& dimension,
+      size_t fNum);
+
+  /**
+   * The operator used in the AGE-MOEA survival score based sorting.
+   *
+   * If a candidate has a lower rank then it is preferred.
+   * Otherwise, if the ranks are equal then the candidate with the larger
+   * Survival Score is preferred.
+   *
+   * @param idxP The index of the first cadidate from the elite population being
+   *     sorted.
+   * @param idxQ The index of the second cadidate from the elite population
+   *     being sorted.
+   * @param ranks The previously calculated ranks.
+   * @param survivalScore The Survival score for each individual in
+   *	the population.
+   * @return true if the first candidate is preferred, otherwise, false.
+   */
+  template<typename MatType>
+  bool SurvivalScoreOperator(
+      size_t idxP,
+      size_t idxQ,
+      const std::vector<size_t>& ranks,
+      const std::vector<typename MatType::elem_type>& survivalScore);
+  
+ /**
+  * Normalizes the front given the extreme points in the current front.
+  *
+  * @tparam The type of population datapoints.
+  * @param calculatedObjectives The current population evaluated objectives.
+  * @param normalization The normalizing vector.
+  * @param front The previously generated Pareto front.
+  * @param extreme The indexes of the extreme points in the front.
+  */
+ template <typename MatType>
+ void NormalizeFront(
+     std::vector<arma::Col<typename MatType::elem_type>>& calculatedObjectives,
+     arma::Col<typename MatType::elem_type>& normalization,
+     const std::vector<size_t>& front,
+     const arma::Row<size_t>& extreme);
+ 
+ /**
+  * Get the geometry information p of Lp norm (p > 0).
+  *
+  * @param calculatedObjectives The current population evaluated objectives.
+  * @param front The previously generated Pareto fronts.
+  * @param extreme The indexes of the extreme points in the front.
+  * @return The variable p in the Lp norm that best fits the geometry of the current front.
+  */
+ template <typename MatType>
+ double GetGeometry(
+      std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+      const std::vector<size_t>& front,
+      const arma::Row<size_t>& extreme);
+  
+  /**
+   * Finds the pairwise Lp distance between all the points in the front.
+   *
+   * @param final The current population evaluated objectives.
+   * @param calculatedObjectives The current population evaluated objectives.
+   * @param front The front of the current generation.
+   * @param dimension The calculated dimension of the front.
+   */
+  template <typename MatType>
+  void PairwiseDistance(
+      MatType& final,
+      std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+      const std::vector<size_t>& front,
+      double dimension);
+
+  /**
+   * Finding the indexes of the extreme points in the front.
+   *  
+   * @param indexes vector containing the slected indexes.
+   * @param calculatedObjectives The current population objectives.
+   * @param front The front of the current generation.
+   */
+  template <typename MatType>
+  void FindExtremePoints(
+      arma::Row<size_t>& indexes,
+      std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+      const std::vector<size_t>& front);
+  
+  /**
+   * Finding the distance of each point in the front from the line formed
+   * by pointA and pointB.
+   * 
+   * @param distance The vector containing the distances of the points in the fron from the line.
+   * @param calculatedObjectives Reference to the current population evaluated Objectives.
+   * @param front The front of the current generation(indices of population).
+   * @param pointA The first point on the line.
+   * @param pointB The second point on the line.
+  */
+  template <typename MatType>
+  void PointToLineDistance(
+      arma::Row<typename MatType::elem_type>& distances,
+      std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+      const std::vector<size_t>& front,
+      const arma::Col<typename MatType::elem_type>& pointA,
+      const arma::Col<typename MatType::elem_type>& pointB);
+  
+  /**
+   * Find the Diversity score corresponding the solution S using the selected set.
+   * 
+   * @param selected The current selected set.
+   * @param pairwiseDistance The current pairwise distance for the whole front.
+   * @param S The relative index of S being considered within the front.
+   * @return The diversity score for S which the sum of the two smallest elements.
+  */
+ template <typename MatType>
+ typename MatType::elem_type DiversityScore(std::set<size_t>& selected,
+                                            const MatType& pairwiseDistance,
+                                            size_t S);
+
+  //! The number of objectives being optimised for.
+  size_t numObjectives;
+
+  //! The numbeer of variables used per objectives.
+  size_t numVariables;
+
+  //! The number of candidates in the population.
+  size_t populationSize;
+
+  //! Maximum number of generations before termination criteria is met.
+  size_t maxGenerations;
+
+  //! Probability that crossover will occur.
+  double crossoverProb;
+
+  //! Probability that mutation will occur.
+  double mutationProb;
+
+  //! Strength of the mutation.
+  double mutationStrength;
+
+  //! The crowding degree of the mutation. Higher value produces a mutant
+  //! resembling its parent.
+  double distributionIndex;
+
+  //! The tolerance for termination.
+  double epsilon;
+
+  //! The distance parameters of the crossover distribution.
+  double eta;
+
+  //! Lower bound of the initial swarm.
+  arma::vec lowerBound;
+
+  //! Upper bound of the initial swarm.
+  arma::vec upperBound;
+
+  //! The set of all the Pareto optimal points.
+  //! Stored after Optimize() is called.
+  arma::cube paretoSet;
+
+  //! The set of all the Pareto optimal objective vectors.
+  //! Stored after Optimize() is called.
+  arma::cube paretoFront;
+
+  //! A different representation of the Pareto front, for reverse compatibility
+  //! purposes.  This can be removed when ensmallen 3.x is released!  (Along
+  //! with `Front()`.)  This is only populated when `Front()` is called.
+  std::vector<arma::mat> rcFront;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "agemoea_impl.hpp"
+
+#endif
@@ -0,0 +1,819 @@
+/**
+ * @file agemoea_impl.hpp
+ * @author Satyam Shukla
+ *
+ * Implementation of the AGEMOEA algorithm. Used for multi-objective
+ * optimization problems on arbitrary functions.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more Information.
+ */
+
+#ifndef ENSMALLEN_AGEMOEA_AGEMOEA_IMPL_HPP
+#define ENSMALLEN_AGEMOEA_AGEMOEA_IMPL_HPP
+
+#include "agemoea.hpp"
+#include <assert.h>
+
+namespace ens {
+
+inline AGEMOEA::AGEMOEA(const size_t populationSize,
+                        const size_t maxGenerations,
+                        const double crossoverProb,
+                        const double distributionIndex,
+                        const double epsilon,
+                        const double eta,
+                        const arma::vec& lowerBound,
+                        const arma::vec& upperBound) :
+    numObjectives(0),
+    numVariables(0),
+    populationSize(populationSize),
+    maxGenerations(maxGenerations),
+    crossoverProb(crossoverProb),
+    distributionIndex(distributionIndex),
+    epsilon(epsilon),
+    eta(eta),
+    lowerBound(lowerBound),
+    upperBound(upperBound)
+{ /* Nothing to do here. */ }
+
+inline AGEMOEA::AGEMOEA(const size_t populationSize,
+                        const size_t maxGenerations,
+                        const double crossoverProb,
+                        const double distributionIndex,
+                        const double epsilon,
+                        const double eta,
+                        const double lowerBound,
+                        const double upperBound) :
+    numObjectives(0),
+    numVariables(0),
+    populationSize(populationSize),
+    maxGenerations(maxGenerations),
+    crossoverProb(crossoverProb),
+    distributionIndex(distributionIndex),
+    epsilon(epsilon),
+    eta(eta),
+    lowerBound(lowerBound * arma::ones(1, 1)),
+    upperBound(upperBound * arma::ones(1, 1))
+{ /* Nothing to do here. */ }
+
+//! Optimize the function.
+template<typename MatType,
+         typename... ArbitraryFunctionType,
+         typename... CallbackTypes>
+typename MatType::elem_type AGEMOEA::Optimize(
+    std::tuple<ArbitraryFunctionType...>& objectives,
+    MatType& iterateIn,
+    CallbackTypes&&... callbacks)
+{
+  // Make sure for evolution to work at least four candidates are present.
+  if (populationSize < 4 && populationSize % 4 != 0)
+  {
+    throw std::logic_error("AGEMOEA::Optimize(): population size should be at"
+        " least 4, and, a multiple of 4!");
+  }
+
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  // Make sure that we have the methods that we need.  Long name...
+  traits::CheckArbitraryFunctionTypeAPI<ArbitraryFunctionType...,
+      BaseMatType>();
+  RequireDenseFloatingPointType<BaseMatType>();
+
+  // Check if lower bound is a vector of a single dimension.
+  if (lowerBound.n_rows == 1)
+    lowerBound = lowerBound(0, 0) * arma::ones(iterate.n_rows, iterate.n_cols);
+
+  // Check if upper bound is a vector of a single dimension.
+  if (upperBound.n_rows == 1)
+    upperBound = upperBound(0, 0) * arma::ones(iterate.n_rows, iterate.n_cols);
+
+  // Check the dimensions of lowerBound and upperBound.
+  assert(lowerBound.n_rows == iterate.n_rows && "The dimensions of "
+      "lowerBound are not the same as the dimensions of iterate.");
+  assert(upperBound.n_rows == iterate.n_rows && "The dimensions of "
+      "upperBound are not the same as the dimensions of iterate.");
+
+  numObjectives = sizeof...(ArbitraryFunctionType);
+  numVariables = iterate.n_rows;
+
+  // Cache calculated objectives.
+  std::vector<arma::Col<ElemType> > calculatedObjectives(populationSize);
+
+  // Population size reserved to 2 * populationSize + 1 to accommodate
+  // for the size of intermediate candidate population.
+  std::vector<BaseMatType> population;
+  population.reserve(2 * populationSize + 1);
+
+  // Pareto fronts, initialized during non-dominated sorting.
+  // Stores indices of population belonging to a certain front.
+  std::vector<std::vector<size_t> > fronts;
+  // Initialised in SurvivalScoreAssignment.
+  std::vector<ElemType> survivalScore;
+  // Initialised during non-dominated sorting.
+  std::vector<size_t> ranks;
+
+  //! Useful temporaries for float-like comparisons.
+  const BaseMatType castedLowerBound = arma::conv_to<BaseMatType>::from(lowerBound);
+  const BaseMatType castedUpperBound = arma::conv_to<BaseMatType>::from(upperBound);
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
+  // Generate the population based on a uniform distribution around the given
+  // starting point.
+  for (size_t i = 0; i < populationSize; i++)
+  {
+    population.push_back(arma::randu<BaseMatType>(iterate.n_rows,
+        iterate.n_cols) - 0.5 + iterate);
+
+    // Constrain all genes to be within bounds.
+    population[i] = arma::min(arma::max(population[i], castedLowerBound), 
+        castedUpperBound);
+  }
+
+  Info << "AGEMOEA initialized successfully. Optimization started." << std::endl;
+
+  // Iterate until maximum number of generations is obtained.
+  Callback::BeginOptimization(*this, objectives, iterate, callbacks...);
+
+  for (size_t generation = 1; generation <= maxGenerations && !terminate; generation++)
+  {
+    // Create new population of candidate from the present elite population.
+    // Have P_t, generate G_t using P_t.
+    BinaryTournamentSelection(population, castedLowerBound, castedUpperBound);
+
+    // Evaluate the objectives for the new population.
+    calculatedObjectives.resize(population.size());
+    std::fill(calculatedObjectives.begin(), calculatedObjectives.end(),
+        arma::Col<ElemType>(numObjectives, arma::fill::zeros));
+    EvaluateObjectives(population, objectives, calculatedObjectives);
+
+    // Perform fast non dominated sort on P_t ∪ G_t.
+    ranks.resize(population.size());
+    FastNonDominatedSort<BaseMatType>(fronts, ranks, calculatedObjectives);
+    
+    arma::Col<ElemType> idealPoint(calculatedObjectives[fronts[0][0]]);
+    for (size_t index = 1; index < fronts[0].size(); index++)
+    {
+      idealPoint = arma::min(idealPoint, 
+          calculatedObjectives[fronts[0][index]]);
+    }
+
+    // Perform survival score assignment.
+    survivalScore.resize(population.size());
+    std::fill(survivalScore.begin(), survivalScore.end(), 0.);
+    double dimension;
+    arma::Col<typename MatType::elem_type> normalize(numObjectives, 
+        arma::fill::zeros);
+    for (size_t fNum = 0; fNum < fronts.size(); fNum++)
+    {
+      SurvivalScoreAssignment<BaseMatType>(fronts[fNum], idealPoint,
+      calculatedObjectives, survivalScore, normalize, dimension, fNum);
+    }
+
+    // Sort based on survival score.
+    std::sort(population.begin(), population.end(),
+      [this, ranks, survivalScore, population]
+        (BaseMatType candidateP, BaseMatType candidateQ)
+          {
+            size_t idxP{}, idxQ{};
+            for (size_t i = 0; i < population.size(); i++)
+            {
+              if (arma::approx_equal(population[i], candidateP, 
+                  "absdiff", epsilon))
+                idxP = i;
+
+              if (arma::approx_equal(population[i], candidateQ, 
+                  "absdiff", epsilon))
+                idxQ = i;
+            }
+
+            return SurvivalScoreOperator<BaseMatType>(idxP, idxQ, ranks, 
+                survivalScore);
+          }
+    );
+
+    // Yield a new population P_{t+1} of size populationSize.
+    // Discards unfit population from the R_{t} to yield P_{t+1}.
+    population.resize(populationSize);
+
+    terminate |= Callback::GenerationalStepTaken(*this, objectives, iterate,
+        calculatedObjectives, fronts, callbacks...);
+  }
+  EvaluateObjectives(population, objectives, calculatedObjectives);
+  // Set the candidates from the Pareto Set as the output.
+  paretoSet.set_size(population[0].n_rows, population[0].n_cols, 
+      population.size());
+  // The Pareto Set is stored, can be obtained via ParetoSet() getter.
+  for (size_t solutionIdx = 0; solutionIdx < population.size(); ++solutionIdx)
+  {
+    paretoSet.slice(solutionIdx) =
+      arma::conv_to<arma::mat>::from(population[solutionIdx]);
+  }
+
+  // Set the candidates from the Pareto Front as the output.
+  paretoFront.set_size(calculatedObjectives[0].n_rows, 
+      calculatedObjectives[0].n_cols, population.size());
+  // The Pareto Front is stored, can be obtained via ParetoFront() getter.
+  for (size_t solutionIdx = 0; solutionIdx < population.size(); ++solutionIdx)
+  {
+    paretoFront.slice(solutionIdx) =
+      arma::conv_to<arma::mat>::from(calculatedObjectives[solutionIdx]);
+  }
+
+  // Clear rcFront, in case it is later requested by the user for reverse
+  // compatibility reasons.
+  rcFront.clear();
+
+  // Assign iterate to first element of the Pareto Set.
+  iterate = population[fronts[0][0]];
+
+  Callback::EndOptimization(*this, objectives, iterate, callbacks...);
+
+  ElemType performance = std::numeric_limits<ElemType>::max();
+
+  for (const arma::Col<ElemType>& objective: calculatedObjectives)
+    if (arma::accu(objective) < performance)
+      performance = arma::accu(objective);
+
+  return performance;
+}
+
+//! No objectives to evaluate.
+template<std::size_t I,
+         typename MatType,
+         typename ...ArbitraryFunctionType>
+typename std::enable_if<I == sizeof...(ArbitraryFunctionType), void>::type
+AGEMOEA::EvaluateObjectives(
+    std::vector<MatType>&,
+    std::tuple<ArbitraryFunctionType...>&,
+    std::vector<arma::Col<typename MatType::elem_type> >&)
+{
+  // Nothing to do here.
+}
+
+//! Evaluate the objectives for the entire population.
+template<std::size_t I,
+         typename MatType,
+         typename ...ArbitraryFunctionType>
+typename std::enable_if<I < sizeof...(ArbitraryFunctionType), void>::type
+AGEMOEA::EvaluateObjectives(
+    std::vector<MatType>& population,
+    std::tuple<ArbitraryFunctionType...>& objectives,
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives)
+{
+  for (size_t i = 0; i < population.size(); i++)
+  {
+    calculatedObjectives[i](I) = std::get<I>(objectives).Evaluate(population[i]);
+    EvaluateObjectives<I+1, MatType, ArbitraryFunctionType...>(population, objectives,
+        calculatedObjectives);
+  }
+}
+
+//! Reproduce and generate new candidates.
+template<typename MatType>
+inline void AGEMOEA::BinaryTournamentSelection(std::vector<MatType>& population,
+                                             const MatType& lowerBound,
+                                             const MatType& upperBound)
+{
+  std::vector<MatType> children;
+
+  while (children.size() < population.size())
+  {
+    // Choose two random parents for reproduction from the elite population.
+    size_t indexA = arma::randi<size_t>(arma::distr_param(0, populationSize - 1));
+    size_t indexB = arma::randi<size_t>(arma::distr_param(0, populationSize - 1));
+
+    // Make sure that the parents differ.
+    if (indexA == indexB)
+    {
+      if (indexB < populationSize - 1)
+        indexB++;
+      else
+        indexB--;
+    }
+
+    // Initialize the children to the respective parents.
+    MatType childA = population[indexA], childB = population[indexB];
+
+    if (arma::randu() <= crossoverProb)
+      Crossover(childA, childB, population[indexA], population[indexB], 
+                lowerBound, upperBound);
+
+    Mutate(childA, 1.0 / static_cast<double>(numVariables),
+        lowerBound, upperBound);
+    Mutate(childB, 1.0 / static_cast<double>(numVariables),
+        lowerBound, upperBound);
+
+    // Add the children to the candidate population.
+    children.push_back(childA);
+    children.push_back(childB);
+  }
+
+  // Add the candidates to the elite population.
+  population.insert(std::end(population), std::begin(children), std::end(children));
+}
+
+//! Perform simulated binary crossover (SBX) of genes for the children.
+template<typename MatType>
+inline void AGEMOEA::Crossover(MatType& childA,
+                               MatType& childB,
+                               const MatType& parentA,
+                               const MatType& parentB,
+                               const MatType& lowerBound,
+                               const MatType& upperBound)
+{
+    //! Generates a child from two parent individuals
+    // according to the polynomial probability distribution.
+    arma::Cube<typename MatType::elem_type> parents(parentA.n_rows, 
+      parentA.n_cols, 2);
+    parents.slice(0) = parentA;
+    parents.slice(1) = parentB;
+    MatType current_min =  arma::min(parents, 2);
+    MatType current_max =  arma::max(parents, 2);
+
+    if (arma::accu(parentA - parentB < 1e-14))
+    {
+      childA = parentA;
+      childB = parentB;
+      return;
+    }
+    MatType current_diff = current_max - current_min;
+    current_diff.transform( [](typename MatType::elem_type val)
+      { return (val < 1e-10 ? 1e-10:val); } );
+
+    // Calculating beta used for the final crossover.
+    MatType beta1 = 1 + 2.0 * (current_min - lowerBound) / current_diff;
+    MatType beta2 = 1 + 2.0 * (upperBound - current_max) / current_diff;
+    MatType alpha1 = 2 - arma::pow(beta1, -(eta + 1));
+    MatType alpha2 = 2 - arma::pow(beta2, -(eta + 1));
+
+    MatType us(arma::size(alpha1), arma::fill::randu);
+    arma::umat mask1 = us > (1.0 / alpha1); 
+    MatType betaq1 = arma::pow(us % alpha1, 1. / (eta + 1));
+    betaq1 = betaq1 % (mask1 != 1.0) + arma::pow((1.0 / (2.0 - us % alpha1)), 
+        1.0 / (eta + 1)) % mask1;
+    arma::umat mask2 = us > (1.0 / alpha2);
+    MatType betaq2 = arma::pow(us % alpha2, 1 / (eta + 1));
+    betaq2 = betaq2 % (mask1 != 1.0) + arma::pow((1.0 / (2.0 - us % alpha2)), 
+        1.0 / (eta + 1)) % mask2;
+
+    // Variables after the cross over for all of them.
+    MatType c1 = 0.5 * ((current_min + current_max) - betaq1 % current_diff);
+    MatType c2 = 0.5 * ((current_min + current_max) + betaq2 % current_diff);
+    c1 = arma::min(arma::max(c1, lowerBound), upperBound);
+    c2 = arma::min(arma::max(c2, lowerBound), upperBound);
+    
+    // Decision for the crossover between the two parents for each variable.
+    us.randu();
+    childA = parentA % (us <= 0.5);
+    childB = parentB % (us <= 0.5);
+    us.randu();
+    childA = childA + c1 % ((us <= 0.5) % (childA == 0));
+    childA = childA + c2 % ((us > 0.5) % (childA == 0));
+    childB = childB + c2 % ((us <= 0.5) % (childB == 0));
+    childB = childB + c1 % ((us > 0.5) % (childB == 0));
+}
+
+//! Perform Polynomial mutation of the candidate.
+template<typename MatType>
+inline void AGEMOEA::Mutate(MatType& candidate,
+                            double mutationRate,
+                            const MatType& lowerBound,
+                            const MatType& upperBound)
+{
+    const size_t numVariables = candidate.n_rows;
+    for (size_t geneIdx = 0; geneIdx < numVariables; ++geneIdx)
+    {
+      // Should this gene be mutated?
+      if (arma::randu() > mutationRate)
+        continue;
+
+      const double geneRange = upperBound(geneIdx) - lowerBound(geneIdx);
+      // Normalised distance from the bounds.
+      const double lowerDelta = (candidate(geneIdx) 
+          - lowerBound(geneIdx)) / geneRange;
+      const double upperDelta = (upperBound(geneIdx) 
+          - candidate(geneIdx)) / geneRange;
+      const double mutationPower = 1. / (distributionIndex + 1.0);
+      const double rand = arma::randu();
+      double value, perturbationFactor;
+      if (rand < 0.5)
+      {
+        value = 2.0 * rand + (1.0 - 2.0 * rand) *
+            std::pow(upperDelta, distributionIndex + 1.0);
+        perturbationFactor = std::pow(value, mutationPower) - 1.0;
+      }
+      else
+      {
+        value = 2.0 * (1.0 - rand) + 2.0 *(rand - 0.5) *
+            std::pow(lowerDelta, distributionIndex + 1.0);
+        perturbationFactor = 1.0 - std::pow(value, mutationPower);
+      }
+
+      candidate(geneIdx) += perturbationFactor * geneRange;
+    }
+    //! Enforce bounds.
+    candidate = arma::min(arma::max(candidate, lowerBound), upperBound);
+}
+
+template <typename MatType>
+inline void AGEMOEA::NormalizeFront(
+      std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+      arma::Col<typename MatType::elem_type>& normalization,
+      const std::vector<size_t>& front,
+      const arma::Row<size_t>& extreme)
+{
+  arma::Mat<typename MatType::elem_type> vectorizedObjectives(numObjectives, 
+      front.size());
+  arma::Mat<typename MatType::elem_type> vectorizedExtremes(numObjectives, 
+      extreme.n_elem);
+  for (size_t i = 0; i < front.size(); i++)
+  {
+    vectorizedObjectives.col(i) = calculatedObjectives[front[i]];
+  }
+  for (size_t i = 0; i < extreme.n_elem; i++)
+  {
+    vectorizedExtremes.col(i) = calculatedObjectives[front[extreme[i]]]; 
+  }
+
+  if (front.size() < numObjectives)
+  {
+    normalization = arma::max(vectorizedObjectives, 1);
+    return;
+  }
+  arma::Col<typename MatType::elem_type> temp;
+  arma::uvec unique = arma::find_unique(extreme);
+  if (extreme.n_elem != unique.n_elem)
+  {
+    normalization = arma::max(vectorizedObjectives, 1);
+    return;
+  }
+  arma::Col<typename MatType::elem_type> one(extreme.n_elem, arma::fill::ones);
+  arma::Col<typename MatType::elem_type> hyperplane(numObjectives, arma::fill::zeros);
+  try{
+    hyperplane = arma::solve(
+    vectorizedExtremes.t(), one);
+  }
+  catch(...)
+  {
+    normalization = arma::max(vectorizedObjectives, 1);
+    normalization = normalization + (normalization == 0);
+    return;
+  }
+  if (hyperplane.has_inf() || hyperplane.has_nan() || (arma::accu(hyperplane < 0.0) > 0))
+  {
+    normalization = arma::max(vectorizedObjectives, 1);
+  }
+  else
+  {
+    normalization = 1. / hyperplane;   
+    if (normalization.has_inf() || normalization.has_nan())
+    {    
+      normalization = arma::max(vectorizedObjectives, 1);
+    }
+  }
+  normalization = normalization + (normalization == 0);
+}
+
+template <typename MatType>
+inline double AGEMOEA::GetGeometry(
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+    const std::vector<size_t>& front,
+    const arma::Row<size_t>& extreme)
+{
+  arma::Row<typename MatType::elem_type> d;
+  arma::Col<typename MatType::elem_type> zero(numObjectives, arma::fill::zeros);
+  arma::Col<typename MatType::elem_type> one(numObjectives, arma::fill::ones);
+
+  PointToLineDistance<MatType> (d, calculatedObjectives, front, zero, one);
+
+  for (size_t i = 0; i < extreme.size(); i++)
+  {
+    d[extreme[i]] = arma::datum::inf;
+  }
+  size_t index = arma::index_min(d);
+  double avg = arma::accu(calculatedObjectives[front[index]]) / static_cast<double> (numObjectives); 
+  double p = std::log(numObjectives) / std::log(1.0 / avg);
+  if (p <= 0.1 || std::isnan(p)) 
+    p = 1.0;
+
+  return p;
+}
+
+//! Pairwise distance for each point in the given front.
+template <typename MatType>
+inline void AGEMOEA::PairwiseDistance(
+    MatType& f,
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+    const std::vector<size_t>& front,
+    double dimension)
+{ 
+  for (size_t i = 0; i < front.size(); i++)
+  {
+    for (size_t j = i + 1; j < front.size(); j++)
+    {
+      f(i, j) = std::pow(arma::accu(arma::pow(arma::abs(calculatedObjectives[front[i]] - calculatedObjectives[front[j]]), dimension)), 1.0 / dimension);
+      f(j, i) = f(i, j);
+    }
+  }
+}
+
+//! Find the index of the of the extreme points in the given front.
+template <typename MatType>
+void AGEMOEA::FindExtremePoints(
+    arma::Row<size_t>& indexes, 
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+    const std::vector<size_t>& front)
+{
+  typedef typename MatType::elem_type ElemType;
+  
+  if (numObjectives >= front.size())
+  {
+    indexes = arma::linspace<arma::Row<size_t>>(0, front.size() - 1, front.size());
+    return;
+  }
+
+  arma::Mat<ElemType> W(numObjectives, numObjectives, arma::fill::eye);
+  W = W + 1e-6;
+  std::vector<bool> selected(front.size());
+  arma::Col<ElemType> z(numObjectives, arma::fill::zeros);
+  arma::Row<ElemType> dists;
+  for (size_t i = 0; i < numObjectives; i++)
+  {
+    PointToLineDistance<MatType>(dists, calculatedObjectives, front, z, W.col(i));
+    for (size_t j = 0; j < front.size(); j++)
+      if (selected[j]){dists[j] = arma::datum::inf;}
+    indexes[i] = dists.index_min();
+    selected[dists.index_min()] = true;
+  }
+}
+
+//! Find the distance of a front from a line formed by two points.
+template <typename MatType>
+void AGEMOEA::PointToLineDistance(
+    arma::Row<typename MatType::elem_type>& distances,
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+    const std::vector<size_t>& front,
+    const arma::Col<typename MatType::elem_type>& pointA,
+    const arma::Col<typename MatType::elem_type>& pointB)
+{
+  typedef typename MatType::elem_type ElemType;
+  arma::Row<ElemType> distancesTemp(front.size());
+  arma::Col<ElemType> ba = pointB - pointA; 
+  arma::Col<ElemType> pa;
+
+  for (size_t i = 0; i < front.size(); i++)
+  {
+    size_t ind = front[i];
+ 
+    pa = (calculatedObjectives[ind] - pointA);
+    double t = arma::dot(pa, ba) / arma::dot(ba, ba);
+    distancesTemp[i] = arma::accu(arma::pow((pa - t * ba), 2));
+  }
+  distances = distancesTemp;
+}
+
+//! Sort population into Pareto fronts.
+template<typename MatType>
+inline void AGEMOEA::FastNonDominatedSort(
+    std::vector<std::vector<size_t> >& fronts,
+    std::vector<size_t>& ranks,
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives)
+{
+  std::map<size_t, size_t> dominationCount;
+  std::map<size_t, std::set<size_t> > dominated;
+
+  // Reset and initialize fronts.
+  fronts.clear();
+  fronts.push_back(std::vector<size_t>());
+
+  for (size_t p = 0; p < calculatedObjectives.size(); p++)
+  {
+    dominated[p] = std::set<size_t>();
+    dominationCount[p] = 0;
+
+    for (size_t q = 0; q < calculatedObjectives.size(); q++)
+    {
+      if (Dominates<MatType>(calculatedObjectives, p, q))
+        dominated[p].insert(q);
+      else if (Dominates<MatType>(calculatedObjectives, q, p))
+        dominationCount[p] += 1;
+    }
+
+    if (dominationCount[p] == 0)
+    {
+      ranks[p] = 0;
+      fronts[0].push_back(p);
+    }
+  }
+
+  size_t i = 0;
+
+  while (!fronts[i].empty())
+  {
+    std::vector<size_t> nextFront;
+
+    for (size_t p: fronts[i])
+    {
+      for (size_t q: dominated[p])
+      {
+        dominationCount[q]--;
+
+        if (dominationCount[q] == 0)
+        {
+          ranks[q] = i + 1;
+          nextFront.push_back(q);
+        }
+      }
+    }
+
+    i++;
+    fronts.push_back(nextFront);
+  }
+  // Remove the empty final set.
+  fronts.pop_back();
+}
+
+//! Check if a candidate Pareto dominates another candidate.
+template<typename MatType>
+inline bool AGEMOEA::Dominates(
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+    size_t candidateP,
+    size_t candidateQ)
+{
+  bool allBetterOrEqual = true;
+  bool atleastOneBetter = false;
+  size_t n_objectives = calculatedObjectives[0].n_elem;
+
+  for (size_t i = 0; i < n_objectives; i++)
+  {
+    // P is worse than Q for the i-th objective function.
+    if (calculatedObjectives[candidateP](i) > calculatedObjectives[candidateQ](i))
+      allBetterOrEqual = false;
+
+    // P is better than Q for the i-th objective function.
+    else if (calculatedObjectives[candidateP](i) < 
+        calculatedObjectives[candidateQ](i))
+      atleastOneBetter = true;
+  }
+
+  return allBetterOrEqual && atleastOneBetter;
+}
+
+//! Assign diversity score for a given point and the selected set.
+template <typename MatType>
+inline typename MatType::elem_type AGEMOEA::DiversityScore(
+    std::set<size_t>& selected,
+    const MatType& pairwiseDistance,
+    size_t S)
+{ 
+  typedef typename MatType::elem_type ElemType;
+  ElemType m = arma::datum::inf;
+  ElemType m1 = arma::datum::inf;
+  std::set<size_t>::iterator it;
+  for (it = selected.begin(); it != selected.end(); it++)
+  {
+    if (*it == S){ continue; }
+    if (pairwiseDistance(S, *it) < m) 
+    {
+      m1 = m;
+      m = pairwiseDistance(S, *it);
+    }
+    else if (pairwiseDistance(S, *it) < m1)
+    {
+      m1 = pairwiseDistance(S, *it);
+    }
+  }
+  m1 = (m1 == arma::datum::inf) ? 0 : m1;
+  m = (m == arma::datum::inf) ? 0 : m;
+  return m + m1;
+}
+
+//! Assign survival score for a front of the population.
+template <typename MatType>
+inline void AGEMOEA::SurvivalScoreAssignment(
+    const std::vector<size_t>& front,
+    const arma::Col<typename MatType::elem_type>& idealPoint,
+    std::vector<arma::Col<typename MatType::elem_type>>& calculatedObjectives,
+    std::vector<typename MatType::elem_type>& survivalScore,
+    arma::Col<typename MatType::elem_type>& normalize,
+    double& dimension,
+    size_t fNum)
+{
+  typedef typename MatType::elem_type ElemType;
+
+  // Calculations for the first front.
+  if (fNum == 0)
+  {
+    if (front.size() < numObjectives)
+    {
+      dimension = 1;
+      arma::Row<size_t> extreme(numObjectives, arma::fill::zeros);
+      NormalizeFront<MatType>(calculatedObjectives, normalize, front, extreme);
+      return; 
+    }
+
+    for (size_t index = 0; index < front.size(); index++)
+    {
+      calculatedObjectives[front[index]] = calculatedObjectives[front[index]] 
+          - idealPoint;
+    }
+
+    arma::Row<size_t> extreme(numObjectives, arma::fill::zeros);
+    FindExtremePoints<MatType>(extreme, calculatedObjectives, front);
+    NormalizeFront<MatType>(calculatedObjectives, normalize, front, extreme);
+
+    for (size_t index = 0; index < front.size(); index++)
+    {
+      calculatedObjectives[front[index]] = calculatedObjectives[front[index]] 
+          / normalize;
+    }
+
+    std::set<size_t> selected;
+    std::set<size_t> remaining;
+    
+    // Create the selected and remaining sets.
+    for (size_t index: extreme)
+    { 
+      selected.insert(index);
+      survivalScore[front[index]] = arma::datum::inf;
+    }
+
+    dimension = GetGeometry<MatType>(calculatedObjectives, front,
+                                           extreme);
+    for (size_t i = 0; i < front.size(); i++)
+    {
+      if (selected.count(i) == 0)
+      {
+        remaining.insert(i);
+      }
+    }
+
+    arma::Mat<ElemType> pairwise(front.size(), front.size(), arma::fill::zeros);
+    PairwiseDistance<MatType>(pairwise,calculatedObjectives,front,dimension);
+    arma::Row<typename MatType::elem_type> value(front.size(), 
+        arma::fill::zeros);
+    
+    // Calculate the diversity and proximity score.
+    for (size_t i = 0; i < front.size(); i++)
+    {
+      pairwise.col(i) = pairwise.col(i) / std::pow(arma::accu(arma::pow(
+        arma::abs(calculatedObjectives[front[i]]), dimension)), 1.0 / dimension);
+    }
+    
+    while (remaining.size() > 0)
+    {
+      std::set<size_t>::iterator it;
+      value = value.fill(-1);
+      for (it = remaining.begin(); it != remaining.end(); it++)
+      {
+        value[*it] = DiversityScore<MatType>(selected, pairwise, *it);
+      }
+      size_t index = arma::index_max(value);
+      survivalScore[front[index]] = value[index];
+      selected.insert(index);
+      remaining.erase(index);
+    }
+  }
+
+  // Calculations for the other fronts.
+  else
+  {
+    for (size_t i = 0; i < front.size(); i++)
+    {
+      calculatedObjectives[front[i]] = (calculatedObjectives[front[i]]) / normalize;
+      survivalScore[front[i]] =  1.0 / std::pow(arma::accu(arma::pow(arma::abs(
+          calculatedObjectives[front[i]] - idealPoint), dimension)), 
+              1.0 / dimension);
+    }
+
+  }
+}
+
+//! Comparator for survival score based sorting.
+template<typename MatType>
+inline bool AGEMOEA::SurvivalScoreOperator(
+    size_t idxP,
+    size_t idxQ,
+    const std::vector<size_t>& ranks,
+    const std::vector<typename MatType::elem_type>& survivalScore)
+{
+  if (ranks[idxP] < ranks[idxQ])
+    return true;
+  else if (ranks[idxP] == ranks[idxQ] && survivalScore[idxP] > survivalScore[idxQ])
+    return true;
+
+  return false;
+}
+
+} // namespace ens
+
+#endif
@@ -117,8 +117,8 @@ AugLagrangian::Optimize(
  for (size_t i = 0; i < function.NumConstraints(); i++)
  {
    const ElemType p = std::pow(function.EvaluateConstraint(i, coordinates), 2);
-    Callback::EvaluateConstraint(*this, function, coordinates, i, p,
-          callbacks...);
+    terminate |= Callback::EvaluateConstraint(*this, function, coordinates, i,
+        p, callbacks...);

    penalty += p;
  }
@@ -129,21 +129,19 @@ AugLagrangian::Optimize(
  // The odd comparison allows user to pass maxIterations = 0 (i.e. no limit on
  // number of iterations).
  size_t it;
-  terminate |= Callback::BeginOptimization(*this, function, coordinates,
-      callbacks...);
+  Callback::BeginOptimization(*this, function, coordinates, callbacks...);
  for (it = 0; it != (maxIterations - 1) && !terminate; it++)
  {
    Info << "AugLagrangian on iteration " << it
        << ", starting with objective "  << lastObjective << "." << std::endl;

    if (!lbfgs.Optimize(augfunc, coordinates, callbacks...))
-      Info << "L-BFGS reported an error during optimization."
-          << std::endl;
-    Info << "Done with L-BFGS: " << coordinates << "\n";
+      Info << "L-BFGS reported an error during optimization." << std::endl;
+    Info << "Done with L-BFGS." << std::endl;

    const ElemType objective = function.Evaluate(coordinates);

-    Callback::Evaluate(*this, function, coordinates, objective,
+    terminate |= Callback::Evaluate(*this, function, coordinates, objective,
        callbacks...);

    // Check if we are done with the entire optimization (the threshold we are
@@ -170,14 +168,17 @@ AugLagrangian::Optimize(
    {
      const ElemType p = std::pow(function.EvaluateConstraint(i, coordinates),
          2);
-      Callback::EvaluateConstraint(*this, function, coordinates, i, p,
-          callbacks...);
+      terminate |= Callback::EvaluateConstraint(*this, function, coordinates, i,
+          p, callbacks...);

      penalty += p;
    }

-    Info << "Penalty is " << penalty << " (threshold "
-        << penaltyThreshold << ")." << std::endl;
+    Info << "Penalty is " << penalty << " (threshold " << penaltyThreshold
+        << ")." << std::endl;
+
+    if (terminate)
+      break;

    if (penalty < penaltyThreshold) // We update lambda.
    {
@@ -186,8 +187,8 @@ AugLagrangian::Optimize(
      for (size_t i = 0; i < function.NumConstraints(); i++)
      {
        const ElemType p = function.EvaluateConstraint(i, coordinates);
-        Callback::EvaluateConstraint(*this, function, coordinates, i, p,
-          callbacks...);
+        terminate |= Callback::EvaluateConstraint(*this, function, coordinates,
+            i, p, callbacks...);

        augfunc.Lambda()[i] -= augfunc.Sigma() * p;
      }
@@ -104,7 +104,7 @@ BigBatchSGD<UpdatePolicyType>::Optimize(
  BaseGradType functionGradient(iterate.n_rows, iterate.n_cols);
  const size_t actualMaxIterations = (maxIterations == 0) ?
      std::numeric_limits<size_t>::max() : maxIterations;
-  terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  Callback::BeginOptimization(*this, f, iterate, callbacks...);
  for (size_t i = 0; i < actualMaxIterations && !terminate;
      /* incrementing done manually */)
  {
@@ -191,6 +191,9 @@ BigBatchSGD<UpdatePolicyType>::Optimize(
      }
    }

+    if (terminate)
+      break;
+
    instUpdatePolicy.As<InstUpdatePolicyType>().Update(f, stepSize, iterate,
        gradient, gB, vB, currentFunction, batchSize, effectiveBatchSize,
        reset);
@@ -229,9 +232,7 @@ BigBatchSGD<UpdatePolicyType>::Optimize(
        return overallObjective;
      }

-      if (std::abs(lastObjective - overallObjective) < tolerance ||
-          Callback::BeginEpoch(*this, f, iterate, epoch, overallObjective,
-              callbacks...))
+      if (std::abs(lastObjective - overallObjective) < tolerance)
      {
        Info << "Big-batch SGD: minimized within tolerance " << tolerance
            << "; terminating optimization." << std::endl;
@@ -240,6 +241,9 @@ BigBatchSGD<UpdatePolicyType>::Optimize(
        return overallObjective;
      }

+      terminate |= Callback::BeginEpoch(*this, f, iterate, epoch,
+          overallObjective, callbacks...);
+
      // Reset the counter variables.
      lastObjective = overallObjective;
      overallObjective = 0;
@@ -250,8 +254,11 @@ BigBatchSGD<UpdatePolicyType>::Optimize(
    }
  }

-  Info << "Big-batch SGD: maximum iterations (" << maxIterations << ") "
-      << "reached; terminating optimization." << std::endl;
+  if (!terminate)
+  {
+    Info << "Big-batch SGD: maximum iterations (" << maxIterations << ") "
+        << "reached; terminating optimization." << std::endl;
+  }

  // Calculate final objective if exactObjective is set to true.
  if (exactObjective)
@@ -263,7 +270,9 @@ BigBatchSGD<UpdatePolicyType>::Optimize(
      const ElemType objective = f.Evaluate(iterate, i, effectiveBatchSize);
      overallObjective += objective;

-      Callback::Evaluate(*this, f, iterate, objective, callbacks...);
+      // The optimization is finished, so we don't need to care what the
+      // callback returns.
+      (void) Callback::Evaluate(*this, f, iterate, objective, callbacks...);
    }
  }

@@ -20,32 +20,35 @@ namespace ens {
 * Callbacks are a set of functions that can be applied at given stages of the
 * optimization process. The following callbacks are available:
 *
- * - Evaluate(optimizer, function, coordinates, objective):
+ * - bool Evaluate(optimizer, function, coordinates, objective):
 *   called after any call to Evaluate().
 *
- * - StepTaken(optimizer, function, coordinates):
+ * - bool StepTaken(optimizer, function, coordinates):
 *   called after any step is taken.
 *
- * - Gradient(optimizer, function, coordinates, gradient):
+ * - bool Gradient(optimizer, function, coordinates, gradient):
 *   called whenever the gradient is computed.
 *
- * - BeginEpoch(optimizer, function, coordinates, epoch, objective):
+ * - bool BeginEpoch(optimizer, function, coordinates, epoch, objective):
 *   called at the beginning of a pass over the data. The objective may be
 *   exact or an estimate depending on exactObjective's value.
 *
- * - EvaluateConstraint(optimizer, function, coordinates, constraint,
- *                      constraintValue):
+ * - bool EvaluateConstraint(optimizer, function, coordinates, constraint,
+ *                           constraintValue):
 *   called after any call to EvaluateConstraint().
 *
- * - GradientConstraint(optimizer, function, coordinates, constraint,
- *                      constraintGradient):
+ * - bool GradientConstraint(optimizer, function, coordinates, constraint,
+ *                           constraintGradient):
 *   called after any call to GradientConstraint().
 *
- * - BeginOptimization(optimizer, function, coordinates):
+ * - void BeginOptimization(optimizer, function, coordinates):
 *   called at the beginning of the optimization.
 *
- * - EndOptimization(optimizer, function, coordinates):
+ * - void EndOptimization(optimizer, function, coordinates):
 *   called at the end of the optimization.
+ *
+ * If true is returned to any of the bool-type callbacks, the optimization will
+ * be terminated before any more steps are taken.
 */
 class Callback
 {
@@ -64,14 +67,14 @@ class Callback
           typename MatType>
  static typename std::enable_if<
      callbacks::traits::HasBeginOptimizationSignature<
-      CallbackType, OptimizerType, FunctionType, MatType>::hasBool,
-      bool>::type
+      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      void>::type
  BeginOptimizationFunction(CallbackType& callback,
                            OptimizerType& optimizer,
                            FunctionType& function,
                            MatType& coordinates)
  {
-    return const_cast<CallbackType&>(callback).BeginOptimization(optimizer,
+    (void) const_cast<CallbackType&>(callback).BeginOptimization(optimizer,
        function, coordinates);
  }

@@ -80,32 +83,14 @@ class Callback
           typename FunctionType,
           typename MatType>
  static typename std::enable_if<
-      callbacks::traits::HasBeginOptimizationSignature<
-      CallbackType, OptimizerType, FunctionType, MatType>::hasVoid,
-      bool>::type
-  BeginOptimizationFunction(CallbackType& callback,
-                            OptimizerType& optimizer,
-                            FunctionType& function,
-                            MatType& coordinates)
-  {
-    const_cast<CallbackType&>(callback).BeginOptimization(optimizer, function,
-        coordinates);
-    return false;
-  }
-
-  template<typename CallbackType,
-           typename OptimizerType,
-           typename FunctionType,
-           typename MatType>
-  static typename std::enable_if<
-      callbacks::traits::HasBeginOptimizationSignature<
-      CallbackType, OptimizerType, FunctionType, MatType>::hasNone,
-      bool>::type
+      !callbacks::traits::HasBeginOptimizationSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      void>::type
  BeginOptimizationFunction(CallbackType& /* callback */,
                            OptimizerType& /* optimizer */,
                            FunctionType& /* function */,
                            MatType& /* coordinates */)
-  { return false; }
+  { /* Nothing to do. */ }

  /**
   * Iterate over the callbacks and invoke the BeginOptimization() callback if
@@ -119,18 +104,28 @@ class Callback
  template<typename OptimizerType,
           typename FunctionType,
           typename MatType,
+           typename CallbackType,
           typename... CallbackTypes>
-  static bool BeginOptimization(OptimizerType& optimizer,
+  static void BeginOptimization(OptimizerType& optimizer,
                                FunctionType& function,
                                MatType& coordinates,
-                                CallbackTypes&... callbacks)
+                                CallbackType& callback,
+                                CallbackTypes&... otherCallbacks)
  {
-    // This will return immediately once a callback returns true.
-    bool result = false;
-    (void)std::initializer_list<bool>{ result =
-        result || Callback::BeginOptimizationFunction(callbacks, optimizer,
-            function, coordinates)... };
-     return result;
+    Callback::BeginOptimizationFunction(callback, optimizer, function,
+        coordinates);
+    Callback::BeginOptimization(optimizer, function, coordinates,
+        otherCallbacks...);
+  }
+
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static void BeginOptimization(OptimizerType& /* optimizer */,
+                                FunctionType& /* function */,
+                                MatType& /* coordinates */)
+  {
+    // Base case... no callbacks left.  Nothing to do.
  }

  /**
@@ -147,28 +142,29 @@ class Callback
           typename MatType>
  static typename std::enable_if<callbacks::traits::HasEndOptimizationSignature<
      CallbackType, OptimizerType, FunctionType, MatType>::value,
-      bool>::type
+      void>::type
  EndOptimizationFunction(CallbackType& callback,
                          OptimizerType& optimizer,
                          FunctionType& function,
                          MatType& coordinates)
  {
-    return (const_cast<CallbackType&>(callback).EndOptimization(
-        optimizer, function, coordinates), false);
+    (void) const_cast<CallbackType&>(callback).EndOptimization( optimizer,
+        function, coordinates);
  }

  template<typename CallbackType,
           typename OptimizerType,
           typename FunctionType,
           typename MatType>
-  static typename std::enable_if<!callbacks::traits::HasEndOptimizationSignature<
+  static typename std::enable_if<
+      !callbacks::traits::HasEndOptimizationSignature<
      CallbackType, OptimizerType, FunctionType, MatType>::value,
-      bool>::type
+      void>::type
  EndOptimizationFunction(CallbackType& /* callback */,
                          OptimizerType& /* optimizer */,
                          FunctionType& /* function */,
                          MatType& /* coordinates */)
-  { return false; }
+  { /* Nothing to do. */ }

  /**
   * Iterate over the callbacks and invoke the EndOptimization() callback if it
@@ -182,18 +178,28 @@ class Callback
  template<typename OptimizerType,
           typename FunctionType,
           typename MatType,
+           typename CallbackType,
           typename... CallbackTypes>
-  static bool EndOptimization(OptimizerType& optimizer,
+  static void EndOptimization(OptimizerType& optimizer,
                              FunctionType& function,
                              MatType& coordinates,
-                              CallbackTypes&... callbacks)
+                              CallbackType& callback,
+                              CallbackTypes&... otherCallbacks)
  {
-    // This will return immediately once a callback returns true.
-    bool result = false;
-    (void)std::initializer_list<bool>{ result =
-        result || Callback::EndOptimizationFunction(callbacks, optimizer,
-            function, coordinates)... };
-     return result;
+    Callback::EndOptimizationFunction(callback, optimizer, function,
+        coordinates);
+    Callback::EndOptimization(optimizer, function, coordinates,
+        otherCallbacks...);
+  }
+
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static void EndOptimization(OptimizerType& /* optimizer */,
+                              FunctionType& /* function */,
+                              MatType& /* coordinates */)
+  {
+    // Base case... no callbacks left.  Nothing to do.
  }

  /**
@@ -210,7 +216,7 @@ class Callback
           typename FunctionType,
           typename MatType>
  static typename std::enable_if<callbacks::traits::HasEvaluateSignature<
-      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      CallbackType, OptimizerType, FunctionType, MatType>::hasBool,
      bool>::type
  EvaluateFunction(CallbackType& callback,
                   OptimizerType& optimizer,
@@ -218,16 +224,34 @@ class Callback
                   const MatType& coordinates,
                   const double objective)
  {
-    return (const_cast<CallbackType&>(callback).Evaluate(
-        optimizer, function, coordinates, objective), false);
+    return const_cast<CallbackType&>(callback).Evaluate(optimizer, function,
+        coordinates, objective);
  }

  template<typename CallbackType,
           typename OptimizerType,
           typename FunctionType,
           typename MatType>
-  static typename std::enable_if<!callbacks::traits::HasEvaluateSignature<
-      CallbackType, OptimizerType, FunctionType, MatType>::value,
+  static typename std::enable_if<callbacks::traits::HasEvaluateSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasVoid,
+      bool>::type
+  EvaluateFunction(CallbackType& callback,
+                   OptimizerType& optimizer,
+                   FunctionType& function,
+                   const MatType& coordinates,
+                   const double objective)
+  {
+    const_cast<CallbackType&>(callback).Evaluate(optimizer, function,
+        coordinates, objective);
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<callbacks::traits::HasEvaluateSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasNone,
      bool>::type
  EvaluateFunction(CallbackType& /* callback */,
                   OptimizerType& /* optimizer */,
@@ -280,7 +304,7 @@ class Callback
           typename MatType>
  static typename std::enable_if<
      callbacks::traits::HasEvaluateConstraintSignature<
-      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      CallbackType, OptimizerType, FunctionType, MatType>::hasBool,
      bool>::type
  EvaluateConstraintFunction(CallbackType& callback,
                             OptimizerType& optimizer,
@@ -289,8 +313,8 @@ class Callback
                             const size_t constraint,
                             const double constraintValue)
  {
-    return (const_cast<CallbackType&>(callback).EvaluateConstraint(
-        optimizer, function, coordinates, constraint, constraintValue), false);
+    return const_cast<CallbackType&>(callback).EvaluateConstraint(
+        optimizer, function, coordinates, constraint, constraintValue);
  }

  template<typename CallbackType,
@@ -298,8 +322,28 @@ class Callback
           typename FunctionType,
           typename MatType>
  static typename std::enable_if<
-      !callbacks::traits::HasEvaluateConstraintSignature<
-      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      callbacks::traits::HasEvaluateConstraintSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasVoid,
+      bool>::type
+  EvaluateConstraintFunction(CallbackType& callback,
+                             OptimizerType& optimizer,
+                             FunctionType& function,
+                             const MatType& coordinates,
+                             const size_t constraint,
+                             const double constraintValue)
+  {
+    const_cast<CallbackType&>(callback).EvaluateConstraint(
+        optimizer, function, coordinates, constraint, constraintValue);
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<
+      callbacks::traits::HasEvaluateConstraintSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasNone,
      bool>::type
  EvaluateConstraintFunction(CallbackType& /* callback */,
                             OptimizerType& /* optimizer */,
@@ -356,7 +400,7 @@ class Callback
           typename MatType,
           typename GradType>
  static typename std::enable_if<callbacks::traits::HasGradientSignature<
-      CallbackType, OptimizerType, FunctionType, MatType, GradType>::value,
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::hasBool,
      bool>::type
  GradientFunction(CallbackType& callback,
                   OptimizerType& optimizer,
@@ -364,8 +408,8 @@ class Callback
                   const MatType& coordinates,
                   GradType& gradient)
  {
-    return (const_cast<CallbackType&>(callback).Gradient(
-        optimizer, function, coordinates, gradient), false);
+    return const_cast<CallbackType&>(callback).Gradient(optimizer, function,
+        coordinates, gradient);
  }

  template<typename CallbackType,
@@ -373,8 +417,27 @@ class Callback
           typename FunctionType,
           typename MatType,
           typename GradType>
-  static typename std::enable_if<!callbacks::traits::HasGradientSignature<
-      CallbackType, OptimizerType, FunctionType, MatType, GradType>::value,
+  static typename std::enable_if<callbacks::traits::HasGradientSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::hasVoid,
+      bool>::type
+  GradientFunction(CallbackType& callback,
+                   OptimizerType& optimizer,
+                   FunctionType& function,
+                   const MatType& coordinates,
+                   GradType& gradient)
+  {
+    const_cast<CallbackType&>(callback).Gradient(
+        optimizer, function, coordinates, gradient);
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType>
+  static typename std::enable_if<callbacks::traits::HasGradientSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::hasNone,
      bool>::type
  GradientFunction(CallbackType& /* callback */,
                   OptimizerType& /* optimizer */,
@@ -427,7 +490,7 @@ class Callback
           typename GradType>
  static typename std::enable_if<
      callbacks::traits::HasGradientConstraintSignature<
-      CallbackType, OptimizerType, FunctionType, MatType, GradType>::value,
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::hasBool,
      bool>::type
  GradientConstraintFunction(CallbackType& callback,
                             OptimizerType& optimizer,
@@ -436,8 +499,8 @@ class Callback
                             const size_t constraint,
                             GradType& gradient)
  {
-    return (const_cast<CallbackType&>(callback).GradientConstraint(
-        optimizer, function, coordinates, constraint, gradient), false);
+    return const_cast<CallbackType&>(callback).GradientConstraint(optimizer,
+        function, coordinates, constraint, gradient);
  }

  template<typename CallbackType,
@@ -446,8 +509,29 @@ class Callback
           typename MatType,
           typename GradType>
  static typename std::enable_if<
-      !callbacks::traits::HasGradientConstraintSignature<
-      CallbackType, OptimizerType, FunctionType, MatType, GradType>::value,
+      callbacks::traits::HasGradientConstraintSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::hasVoid,
+      bool>::type
+  GradientConstraintFunction(CallbackType& callback,
+                             OptimizerType& optimizer,
+                             FunctionType& function,
+                             const MatType& coordinates,
+                             const size_t constraint,
+                             GradType& gradient)
+  {
+    const_cast<CallbackType&>(callback).GradientConstraint(
+        optimizer, function, coordinates, constraint, gradient);
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType>
+  static typename std::enable_if<
+      callbacks::traits::HasGradientConstraintSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::hasNone,
      bool>::type
  GradientConstraintFunction(CallbackType& /* callback */,
                             OptimizerType& /* optimizer */,
@@ -539,7 +623,7 @@ class Callback
           typename FunctionType,
           typename MatType>
  static typename std::enable_if<callbacks::traits::HasBeginEpochSignature<
-      CallbackType, OptimizerType, FunctionType, MatType>::value, bool>::type
+      CallbackType, OptimizerType, FunctionType, MatType>::hasBool, bool>::type
  BeginEpochFunction(CallbackType& callback,
                     OptimizerType& optimizer,
                     FunctionType& function,
@@ -547,16 +631,34 @@ class Callback
                     const size_t epoch,
                     const double objective)
  {
-    return (const_cast<CallbackType&>(callback).BeginEpoch(
-        optimizer, function, coordinates, epoch, objective), false);
+    return const_cast<CallbackType&>(callback).BeginEpoch(
+        optimizer, function, coordinates, epoch, objective);
  }

  template<typename CallbackType,
           typename OptimizerType,
           typename FunctionType,
           typename MatType>
-  static typename std::enable_if<!callbacks::traits::HasBeginEpochSignature<
-      CallbackType, OptimizerType, FunctionType, MatType>::value, bool>::type
+  static typename std::enable_if<callbacks::traits::HasBeginEpochSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasVoid, bool>::type
+  BeginEpochFunction(CallbackType& callback,
+                     OptimizerType& optimizer,
+                     FunctionType& function,
+                     const MatType& coordinates,
+                     const size_t epoch,
+                     const double objective)
+  {
+    const_cast<CallbackType&>(callback).BeginEpoch(
+        optimizer, function, coordinates, epoch, objective);
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<callbacks::traits::HasBeginEpochSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasNone, bool>::type
  BeginEpochFunction(CallbackType& /* callback  */,
                     OptimizerType& /* optimizer */,
                     FunctionType& /* function  */,
@@ -769,6 +871,112 @@ class Callback
            function, coordinates)... };
     return result;
  }
+
+ /**
+  * Invoke the GenerationalStepTaken() callback if it exists.
+  * Specialization for MultiObjective case.
+  *
+  * @param callback The callback to call.
+  * @param optimizer The optimizer used to update the function.
+  * @param function Function to optimize.
+  * @param coordinates Starting point.
+  * @param objectives The set of calculated objectives so far.
+  * @param frontIndices The indices of the members belonging to Pareto Front.
+  */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename ObjectivesVecType,
+           typename IndicesType>
+  static typename std::enable_if<
+      callbacks::traits::HasGenerationalStepTakenSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, ObjectivesVecType,
+      IndicesType>::hasBool, bool>::type
+  GenerationalStepTakenFunction(CallbackType& callback,
+                                OptimizerType& optimizer,
+                                FunctionType& function,
+                                MatType& coordinates,
+                                ObjectivesVecType& objectives,
+                                IndicesType& frontIndices)
+  {
+    return const_cast<CallbackType&>(callback).GenerationalStepTaken(
+        optimizer, function, coordinates, objectives, frontIndices);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename ObjectivesVecType,
+           typename IndicesType>
+  static typename std::enable_if<
+      callbacks::traits::HasGenerationalStepTakenSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, ObjectivesVecType,
+      IndicesType>::hasVoid, bool>::type
+  GenerationalStepTakenFunction(CallbackType& callback,
+                                OptimizerType& optimizer,
+                                FunctionType& function,
+                                MatType& coordinates,
+                                ObjectivesVecType& objectives,
+                                IndicesType& frontIndices)
+  {
+    const_cast<CallbackType&>(callback).GenerationalStepTaken(
+        optimizer, function, coordinates, objectives, frontIndices);
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename ObjectivesVecType,
+           typename IndicesType>
+  static typename std::enable_if<
+      callbacks::traits::HasGenerationalStepTakenSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, ObjectivesVecType,
+      IndicesType>::hasNone, bool>::type
+  GenerationalStepTakenFunction(CallbackType& /* callback */,
+                                OptimizerType& /* optimizer */,
+                                FunctionType& /* function */,
+                                MatType& /* coordinates */,
+                                ObjectivesVecType& /* objectives */,
+                                IndicesType& /* frontIndices */)
+  { return false; }
+
+  /**
+   * Iterate over the callbacks and invoke the GenerationalStepTaken() callback if it
+   * exists.
+   *
+   * Specialization for MultiObjective case.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param objectives The set of calculated objectives so far.
+   * @param frontIndices The indices of the members belonging to Pareto Front.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename ObjectivesVecType,
+           typename IndicesType,
+           typename MatType,
+           typename ...CallbackTypes>
+  static bool GenerationalStepTaken(OptimizerType& optimizer,
+                                    FunctionType& functions,
+                                    MatType& coordinates,
+                                    ObjectivesVecType& objectives,
+                                    IndicesType& frontIndices,
+                                    CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)std::initializer_list<bool>{ result = result ||
+        Callback::GenerationalStepTakenFunction(callbacks, optimizer, functions,
+        coordinates, objectives, frontIndices)... };
+    return result;
+  }
 };

 } // namespace ens
@@ -32,9 +32,9 @@ class EarlyStopAtMinLossType
   * @param patienceIn The number of epochs to wait after the minimum loss has
   *    been reached or no improvement has been made (Default: 10).
   */
-  EarlyStopAtMinLossType<MatType>(const size_t patienceIn = 10) :
-      callbackUsed(false), 
-      patience(patienceIn), 
+  EarlyStopAtMinLossType(const size_t patienceIn = 10) :
+      callbackUsed(false),
+      patience(patienceIn),
      bestObjective(std::numeric_limits<double>::max()),
      steps(0)
  { /* Nothing to do here */ }
@@ -47,13 +47,13 @@ class EarlyStopAtMinLossType
   * @param patienceIn The number of epochs to wait after the minimum loss has
   *    been reached or no improvement has been made (Default: 10).
   */
-  EarlyStopAtMinLossType<MatType>(
+  EarlyStopAtMinLossType(
      std::function<double(const MatType&)> func,
      const size_t patienceIn = 10)
-    : callbackUsed(true), 
-      patience(patienceIn), 
+    : callbackUsed(true),
+      patience(patienceIn),
      bestObjective(std::numeric_limits<double>::max()),
-      steps(0), 
+      steps(0),
      localFunc(func)
  {
    // Nothing to do here
@@ -78,7 +78,7 @@ class EarlyStopAtMinLossType
    if (callbackUsed)
    {
      objective = localFunc(coordinates);
-    } 
+    }

    if (objective < bestObjective)
    {
@@ -98,7 +98,8 @@ class EarlyStopAtMinLossType
  }

 private:
-  //! False if the first constructor is called, true if the user passed a lambda. 
+  //! False if the first constructor is called, true if the user passed a
+  //! lambda.
  bool callbackUsed;

  //! The number of epochs to wait before terminating the optimization process.
@@ -0,0 +1,61 @@
+/**
+ * @file grad_clip_by_norm.hpp
+ * @author Marcus Edel
+ *
+ * Clip the gradients by multiplying the unit vector of the gradients with the
+ * threshold.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_NORM_HPP
+#define ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_NORM_HPP
+
+namespace ens {
+
+/**
+ * Clip the gradients by multiplying the unit vector of the gradients with the
+ * threshold.
+ */
+class GradClipByNorm
+{
+ public:
+  /**
+   * Set up the gradient clip by norm callback class with the maximum clipping
+   * value.
+   *
+   * @param maxNorm The maximum clipping value.
+   */
+  GradClipByNorm(const double maxNorm) : maxNorm(maxNorm)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Callback function called at any call to Gradient().
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param gradient Matrix that holds the gradient.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  bool Gradient(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                MatType& gradient)
+  {
+    const double gradientNorm = arma::norm(gradient);
+    if (gradientNorm > maxNorm)
+      gradient = maxNorm * gradient / gradientNorm;
+    return false;
+  }
+
+ private:
+  //! The maximum clipping value for gradient clipping.
+  const double maxNorm;
+};
+
+} // namespace ens
+
+#endif
@@ -0,0 +1,61 @@
+/**
+ * @file grad_clip_by_value.hpp
+ * @author Marcus Edel
+ *
+ * Clips the gradient to a specified min and max.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_VALUE_HPP
+#define ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_VALUE_HPP
+
+namespace ens {
+
+/**
+ * Clip the gradient to a specified min and max.
+ */
+class GradClipByValue
+{
+ public:
+  /**
+   * Set up the gradient clip by value callback class with the min and max
+   * value.
+   *
+   * @param min The minimum value to clip to.
+   * @param max The maximum value to clip to.
+   */
+  GradClipByValue(const double min, const double max) : lower(min), upper(max)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Callback function called at any call to Gradient().
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param gradient Matrix that holds the gradient.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  bool Gradient(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                MatType& gradient)
+  {
+    gradient = arma::clamp(gradient, lower, upper);
+    return false;
+  }
+
+ private:
+  //! The minimum value to clip to.
+  const double lower;
+
+  //! The maximum value to clip to.
+  const double upper;
+};
+
+} // namespace ens
+
+#endif
@@ -38,13 +38,14 @@ class PrintLoss
   * @param objective Objective value of the current point.
   */
  template<typename OptimizerType, typename FunctionType, typename MatType>
-  void EndEpoch(OptimizerType& /* optimizer */,
+  bool EndEpoch(OptimizerType& /* optimizer */,
                FunctionType& /* function */,
                const MatType& /* coordinates */,
                const size_t /* epoch */,
                const double objective)
  {
    output << objective << std::endl;
+    return false;
  }

 private:
@@ -98,7 +98,7 @@ class ProgressBar
   * @param objective Objective value of the current point.
   */
  template<typename OptimizerType, typename FunctionType, typename MatType>
-  void BeginEpoch(OptimizerType& /* optimizer */,
+  bool BeginEpoch(OptimizerType& /* optimizer */,
                  FunctionType& /* function */,
                  const MatType& /* coordinates */,
                  const size_t epochIn,
@@ -113,6 +113,8 @@ class ProgressBar

    epoch = epochIn;
    newEpoch = true;
+
+    return false;
  }

  /**
@@ -124,7 +126,7 @@ class ProgressBar
   * @param objective Objective value of the current point.
   */
  template<typename OptimizerType, typename FunctionType, typename MatType>
-  void StepTaken(OptimizerType& /* optimizer */,
+  bool StepTaken(OptimizerType& /* optimizer */,
                 FunctionType& /* function */,
                 const MatType& /* coordinates */)
  {
@@ -163,6 +165,8 @@ class ProgressBar
    output.flush();

    stepTimer.tic();
+
+    return false;
  }

  /**
@@ -174,13 +178,14 @@ class ProgressBar
   * @param objectiveIn Objective value of the current point.
   */
  template<typename OptimizerType, typename FunctionType, typename MatType>
-  void Evaluate(OptimizerType& optimizer,
+  bool Evaluate(OptimizerType& optimizer,
                FunctionType& /* function */,
                const MatType& /* coordinates */,
                const double objectiveIn)
  {
    objective += objectiveIn / optimizer.BatchSize();
    steps++;
+    return false;
  }

  /**
@@ -193,7 +198,7 @@ class ProgressBar
   * @param objective Objective value of the current point.
   */
  template<typename OptimizerType, typename FunctionType, typename MatType>
-  void EndEpoch(OptimizerType& /* optimizer */,
+  bool EndEpoch(OptimizerType& /* optimizer */,
                FunctionType& /* function */,
                const MatType& /* coordinates */,
                const size_t /* epoch */,
@@ -216,11 +221,12 @@ class ProgressBar
        output << ".";
      }
    }
-
-    const size_t stepTime = epochTimer.toc() / (double) epochSize * 1000;
-    output << "] " << progress << "% - " << (size_t) epochTimer.toc() % 60
-        << "s " << stepTime << "ms/step " << "- loss: " << objective  <<  "\n";
+    const double epochTimerElapsed = epochTimer.toc();
+    const size_t stepTime = epochTimerElapsed / (double) epochSize * 1000;
+    output << "] " << progress << "% - " << epochTimerElapsed
+        << "s/epoch; " << stepTime << "ms/step; loss: " << objective  <<  "\n";
    output.flush();
+    return false;
  }

 private:
@@ -0,0 +1,89 @@
+/**
+ * @file query_front.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * Implementation of the query front callback function.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_QUERY_FRONT_HPP
+#define ENSMALLEN_CALLBACKS_QUERY_FRONT_HPP
+
+namespace ens {
+
+/**
+ * Query the current Pareto Front after every GenerationalStepTaken callback function.
+ */
+class QueryFront
+{
+ public:
+  /**
+   * Set up the query front callback class with the specified inputs.
+   *
+   * @param queryRate The frequency at which the Pareto Front is queried.
+   * @param paretoFrontArray A reference to a vector of cube to store the
+   *     queried fronts.
+   */
+  QueryFront(const size_t queryRate,
+             std::vector<arma::cube>& paretoFrontArray) :
+      queryRate(queryRate),
+      paretoFrontArray(paretoFrontArray),
+      genCounter(0)
+  { /* Nothing to do here */ }
+
+  /**
+   * Callback function called at the end of a single generational run.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param objectives The set of calculated objectives so far.
+   * @param frontIndices The indices of the members belonging to Pareto Front.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename ObjectivesVecType,
+           typename IndicesType>
+  bool GenerationalStepTaken(OptimizerType& /* opt */,
+                             FunctionType& /* function */,
+                             const MatType& /* coordinates */,
+                             const ObjectivesVecType& objectives,
+                             const IndicesType& frontIndices)
+  {
+    arma::cube currentParetoFront{};
+
+    if (genCounter % queryRate == 0)
+    {
+      currentParetoFront.resize(objectives[0].n_rows, objectives[0].n_cols,
+          frontIndices[0].size());
+      for (size_t solutionIdx = 0; solutionIdx < frontIndices[0].size();
+          ++solutionIdx)
+      {
+        currentParetoFront.slice(solutionIdx) = arma::conv_to<arma::mat>::from(
+            objectives[frontIndices[0][solutionIdx]]);
+      }
+
+      paretoFrontArray.emplace_back(std::move(currentParetoFront));
+    }
+
+    ++genCounter;
+    return false;
+  }
+
+
+ private:
+  //! The rate of query.
+  size_t queryRate;
+  //! A reference to the array of pareto fronts.
+  std::vector<arma::cube>& paretoFrontArray;
+  //! A counter for the current generation.
+  size_t genCounter;
+};
+
+} // namespace ens
+
+#endif
@@ -87,6 +87,7 @@ class Report
      TruncatePrint(initialCoordinates, outputMatrixSize);
      output << std::endl << "Final coordinates: " << std::endl;
      TruncatePrint(coordinates, outputMatrixSize);
+      output << std::endl;
    }
    else
    {
@@ -239,13 +240,14 @@ class Report
   * @param objective Objective value of the current point.
   */
  template<typename OptimizerType, typename FunctionType, typename MatType>
-  void BeginEpoch(OptimizerType& /* optimizer */,
+  bool BeginEpoch(OptimizerType& /* optimizer */,
                  FunctionType& /* function */,
                  const MatType& /* coordinates */,
                  const size_t /* epoch */,
                  const double /* objective */)
  {
    epochCalls++;
+    return false;
  }

  /**
@@ -258,7 +260,7 @@ class Report
   * @param objective Objective value of the current point.
   */
  template<typename OptimizerType, typename FunctionType, typename MatType>
-  void EndEpoch(OptimizerType& optimizer,
+  bool EndEpoch(OptimizerType& optimizer,
                FunctionType& /* function */,
                const MatType& /* coordinates */,
                const size_t /* epoch */,
@@ -282,6 +284,7 @@ class Report
      gradientsNorm.push_back(gradientNorm);

    SaveStepSize(optimizer);
+    return false;
  }

  /**
@@ -293,7 +296,7 @@ class Report
   * @param objective Objective value of the current point.
   */
  template<typename OptimizerType, typename FunctionType, typename MatType>
-  void StepTaken(OptimizerType& optimizer,
+  bool StepTaken(OptimizerType& optimizer,
                 FunctionType& /* function */,
                 const MatType& /* coordinates */)
  {
@@ -307,6 +310,7 @@ class Report

      SaveStepSize(optimizer);
    }
+    return false;
  }

  /**
@@ -318,13 +322,14 @@ class Report
   * @param objectiveIn Objective value of the current point.
   */
  template<typename OptimizerType, typename FunctionType, typename MatType>
-  void Evaluate(OptimizerType& /* optimizer */,
+  bool Evaluate(OptimizerType& /* optimizer */,
                FunctionType& /* function */,
                const MatType& /* coordinates */,
                const double objectiveIn)
  {
    objective = objectiveIn;
    evaluateCalls++;
+    return false;
  }

  /**
@@ -337,7 +342,7 @@ class Report
    * @param objectiveIn Objective value of the current point.
    */
  template<typename OptimizerType, typename FunctionType, typename MatType>
-  void EvaluateConstraint(OptimizerType& /* optimizer */,
+  bool EvaluateConstraint(OptimizerType& /* optimizer */,
                          FunctionType& /* function */,
                          const MatType& /* coordinates */,
                          const size_t /* constraint */,
@@ -345,6 +350,7 @@ class Report
  {
    objective += objectiveIn;
    evaluateCalls++;
+    return false;
  }

  /**
@@ -356,7 +362,7 @@ class Report
   * @param gradientIn Matrix that holds the gradient.
   */
  template<typename OptimizerType, typename FunctionType, typename MatType>
-  void Gradient(OptimizerType& /* optimizer */,
+  bool Gradient(OptimizerType& /* optimizer */,
                FunctionType& /* function */,
                const MatType& /* coordinates */,
                const MatType& gradientIn)
@@ -364,6 +370,7 @@ class Report
    hasGradient = true;
    gradientNorm = arma::norm(gradientIn);
    gradientCalls++;
+    return false;
  }

  /**
@@ -376,13 +383,14 @@ class Report
   * @param gradient Matrix that holds the gradient;
   */
  template<typename OptimizerType, typename FunctionType, typename MatType>
-  void GradientConstraint(OptimizerType& optimizer,
+  bool GradientConstraint(OptimizerType& optimizer,
                          FunctionType& function,
                          const MatType& coordinates,
                          const size_t /* constraint */,
                          const MatType& gradient)
  {
    Gradient(optimizer, function, coordinates, gradient);
+    return false;
  }

 private:
@@ -40,7 +40,7 @@ class StoreBestCoordinates
   * @param objective Objective value of the current point.
   */
  template<typename OptimizerType, typename FunctionType, typename MatType>
-  void Evaluate(OptimizerType& /* optimizer */,
+  bool Evaluate(OptimizerType& /* optimizer */,
                FunctionType& /* function */,
                const MatType& coordinates,
                const double objective)
@@ -50,6 +50,7 @@ class StoreBestCoordinates
      bestObjective = objective;
      bestCoordinates = coordinates;
    }
+    return false;
  }

  //! Get the best coordinates.
@@ -37,6 +37,8 @@ ENS_HAS_EXACT_METHOD_FORM(BeginEpoch, HasBeginEpoch)
 ENS_HAS_EXACT_METHOD_FORM(EndEpoch, HasEndEpoch)
 //! Detect an StepTaken() method.
 ENS_HAS_EXACT_METHOD_FORM(StepTaken, HasStepTaken)
+//! Detect an GenerationalStepTaken() method.
+ENS_HAS_EXACT_METHOD_FORM(GenerationalStepTaken, HasGenerationalStepTaken)

 template<typename OptimizerType,
         typename FunctionType,
@@ -47,7 +49,7 @@ struct TypedForms
  //! This is the form of a bool Evaluate() callback method.
  template<typename CallbackType>
  using EvaluateBoolForm =
-      void(CallbackType::*)(OptimizerType&,
+      bool(CallbackType::*)(OptimizerType&,
                            FunctionType&,
                            const MatType&,
                            const double);
@@ -63,7 +65,7 @@ struct TypedForms
  //! This is the form of a bool EvaluateConstraint() callback method.
  template<typename CallbackType>
  using EvaluateConstraintBoolForm =
-      void(CallbackType::*)(OptimizerType&,
+      bool(CallbackType::*)(OptimizerType&,
                            FunctionType&,
                            const MatType&,
                            const size_t,
@@ -81,11 +83,20 @@ struct TypedForms
  //! This is the form of a bool Gradient() callback method.
  template<typename CallbackType>
  using GradientBoolForm =
-      void(CallbackType::*)(OptimizerType&,
+      bool(CallbackType::*)(OptimizerType&,
                            FunctionType&,
                            const MatType&,
                            const MatType&);

+  //! This is the form of a bool Gradient() callback method where the gradient
+  //! is modifiable.
+  template<typename CallbackType>
+  using GradientBoolModifiableForm =
+      bool(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            MatType&);
+
  //! This is the form of a void Gradient() callback method.
  template<typename CallbackType>
  using GradientVoidForm =
@@ -94,15 +105,34 @@ struct TypedForms
                            const MatType&,
                            const MatType&);

+  //! This is the form of a void Gradient() callback method where the gradient
+  //! is modifiable.
+  template<typename CallbackType>
+  using GradientVoidModifiableForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            MatType&);
+
  //! This is the form of a bool GradientConstraint() callback method.
  template<typename CallbackType>
  using GradientConstraintBoolForm =
-      void(CallbackType::*)(OptimizerType&,
+      bool(CallbackType::*)(OptimizerType&,
                            FunctionType&,
                            const MatType&,
                            const size_t,
                            const MatType&);

+  //! This is the form of a bool GradientConstraint() callback method where the
+  //! gradient is modifiable.
+  template<typename CallbackType>
+  using GradientConstraintBoolModifiableForm =
+      bool(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            MatType&);
+
  //! This is the form of a void GradientConstraint() callback method.
  template<typename CallbackType>
  using GradientConstraintVoidForm =
@@ -112,6 +142,16 @@ struct TypedForms
                            const size_t,
                            const MatType&);

+  //! This is the form of a void GradientConstraint() callback method where the
+  //! gradient is modifiable.
+  template<typename CallbackType>
+  using GradientConstraintVoidModifiableForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            MatType&);
+
  //! This is the form of a bool BeginOptimization() callback method.
  template<typename CallbackType>
  using BeginOptimizationBoolForm =
@@ -199,23 +239,11 @@ template<typename CallbackType,
         typename MatType>
 struct HasBeginOptimizationSignature
 {
-  const static bool hasBool =
+  constexpr static bool value =
+      HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template BeginOptimizationBoolForm>::value ||
      HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
-          FunctionType, MatType>::template BeginOptimizationBoolForm>::value &&
-      !HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
          FunctionType, MatType>::template BeginOptimizationVoidForm>::value;
-
-  const static bool hasVoid =
-      !HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
-          FunctionType, MatType>::template BeginOptimizationBoolForm>::value &&
-      HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
-         FunctionType, MatType>::template BeginOptimizationVoidForm>::value;
-
-  const static bool hasNone =
-      !HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
-          FunctionType, MatType>::template BeginOptimizationBoolForm>::value &&
-      !HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
-         FunctionType, MatType>::template BeginOptimizationVoidForm>::value;
 };

 //! Utility struct, check if either void Evaluate() or bool Evaluate()
@@ -226,11 +254,23 @@ template<typename CallbackType,
         typename MatType>
 struct HasEvaluateSignature
 {
-  const static bool value =
+  constexpr static bool hasBool =
      HasEvaluate<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EvaluateBoolForm>::value ||
+          FunctionType, MatType>::template EvaluateBoolForm>::value &&
+      !HasEvaluate<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EvaluateVoidForm>::value;
+
+  constexpr static bool hasVoid =
+      !HasEvaluate<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EvaluateBoolForm>::value &&
      HasEvaluate<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EvaluateVoidForm>::value;
+          FunctionType, MatType>::template EvaluateVoidForm>::value;
+
+  constexpr static bool hasNone =
+      !HasEvaluate<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EvaluateBoolForm>::value &&
+      !HasEvaluate<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EvaluateVoidForm>::value;
 };

 //! Utility struct, check if either void EvaluateConstraint() or
@@ -241,11 +281,23 @@ template<typename CallbackType,
         typename MatType>
 struct HasEvaluateConstraintSignature
 {
-  const static bool value =
+  constexpr static bool hasBool =
      HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EvaluateConstraintBoolForm>::value ||
+          FunctionType, MatType>::template EvaluateConstraintBoolForm>::value &&
+      !HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EvaluateConstraintVoidForm>::value;
+
+  constexpr static bool hasVoid =
+      !HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EvaluateConstraintBoolForm>::value &&
      HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EvaluateConstraintVoidForm>::value;
+          FunctionType, MatType>::template EvaluateConstraintVoidForm>::value;
+
+  constexpr static bool hasNone =
+      !HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EvaluateConstraintBoolForm>::value &&
+      !HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EvaluateConstraintVoidForm>::value;
 };

 //! Utility struct, check if either void Gradient() or bool Gradient()
@@ -257,11 +309,41 @@ template<typename CallbackType,
         typename Gradient>
 struct HasGradientSignature
 {
-  const static bool value =
+  constexpr static bool hasBool =
+      (HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template GradientBoolForm>::value ||
      HasGradient<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType, Gradient>::template GradientBoolForm>::value ||
+          FunctionType, MatType, Gradient>::template
+          GradientBoolModifiableForm>::value) &&
+      (!HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template GradientVoidForm>::value ||
+      !HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template
+          GradientVoidModifiableForm>::value);
+
+  constexpr static bool hasVoid =
+      (!HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template GradientBoolForm>::value ||
+      !HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template
+          GradientBoolModifiableForm>::value) &&
+      (HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template GradientVoidForm>::value ||
      HasGradient<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType, Gradient>::template GradientVoidForm>::value;
+          FunctionType, MatType, Gradient>::template
+          GradientVoidModifiableForm>::value);
+
+  constexpr static bool hasNone =
+      !HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template GradientBoolForm>::value &&
+      !HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template
+          GradientBoolModifiableForm>::value &&
+      !HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template GradientVoidForm>::value &&
+      !HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template
+          GradientVoidModifiableForm>::value;
 };

 //! Utility struct, check if either void GradientConstraint() or
@@ -273,13 +355,29 @@ template<typename CallbackType,
         typename Gradient>
 struct HasGradientConstraintSignature
 {
-  const static bool value =
+  constexpr static bool hasBool =
      HasGradientConstraint<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType,
-          Gradient>::template GradientConstraintBoolForm>::value ||
+          FunctionType, MatType, Gradient>::template
+          GradientConstraintBoolForm>::value &&
+      !HasGradientConstraint<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template
+          GradientConstraintVoidForm>::value;
+
+  constexpr static bool hasVoid =
+      !HasGradientConstraint<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template
+          GradientConstraintBoolForm>::value &&
      HasGradientConstraint<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType,
-          Gradient>::template GradientConstraintVoidForm>::value;
+          FunctionType, MatType, Gradient>::template
+          GradientConstraintVoidForm>::value;
+
+  constexpr static bool hasNone =
+      !HasGradientConstraint<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template
+          GradientConstraintBoolForm>::value &&
+      !HasGradientConstraint<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template
+          GradientConstraintVoidForm>::value;
 };

 //! Utility struct, check if either void EndOptimization() or
@@ -290,11 +388,11 @@ template<typename CallbackType,
         typename MatType>
 struct HasEndOptimizationSignature
 {
-  const static bool value =
+  constexpr static bool value =
      HasEndOptimization<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EndOptimizationBoolForm>::value ||
+          FunctionType, MatType>::template EndOptimizationBoolForm>::value ||
      HasEndOptimization<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EndOptimizationVoidForm>::value;
+          FunctionType, MatType>::template EndOptimizationVoidForm>::value;
 };

 //! Utility struct, check if either void BeginEpoch() or bool BeginEpoch()
@@ -305,11 +403,23 @@ template<typename CallbackType,
         typename MatType>
 struct HasBeginEpochSignature
 {
-  const static bool value =
+  constexpr static bool hasBool =
      HasBeginEpoch<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template BeginEpochBoolForm>::value ||
+          FunctionType, MatType>::template BeginEpochBoolForm>::value &&
+      !HasBeginEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template BeginEpochVoidForm>::value;
+
+  constexpr static bool hasVoid =
+      !HasBeginEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template BeginEpochBoolForm>::value &&
      HasBeginEpoch<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template BeginEpochVoidForm>::value;
+          FunctionType, MatType>::template BeginEpochVoidForm>::value;
+
+  constexpr static bool hasNone =
+      !HasBeginEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template BeginEpochBoolForm>::value &&
+      !HasBeginEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template BeginEpochVoidForm>::value;
 };

 //! Utility struct, check if either void EndEpoch() or bool EndEpoch()
@@ -320,19 +430,19 @@ template<typename CallbackType,
         typename MatType>
 struct HasEndEpochSignature
 {
-  const static bool hasBool =
+  constexpr static bool hasBool =
      HasEndEpoch<CallbackType, TypedForms<OptimizerType,
          FunctionType, MatType>::template EndEpochBoolForm>::value &&
      !HasEndEpoch<CallbackType, TypedForms<OptimizerType,
          FunctionType, MatType>::template EndEpochVoidForm>::value;

-  const static bool hasVoid =
+  constexpr static bool hasVoid =
      !HasEndEpoch<CallbackType, TypedForms<OptimizerType,
          FunctionType, MatType>::template EndEpochBoolForm>::value &&
      HasEndEpoch<CallbackType, TypedForms<OptimizerType,
          FunctionType, MatType>::template EndEpochVoidForm>::value;

-  const static bool hasNone =
+  constexpr static bool hasNone =
      !HasEndEpoch<CallbackType, TypedForms<OptimizerType,
          FunctionType, MatType>::template EndEpochBoolForm>::value &&
      !HasEndEpoch<CallbackType, TypedForms<OptimizerType,
@@ -346,25 +456,88 @@ template<typename CallbackType,
         typename MatType>
 struct HasStepTakenSignature
 {
-  const static bool hasBool =
+  constexpr static bool hasBool =
      HasStepTaken<CallbackType, TypedForms<OptimizerType,
          FunctionType, MatType>::template StepTakenBoolForm>::value &&
      !HasStepTaken<CallbackType, TypedForms<OptimizerType,
          FunctionType, MatType>::template StepTakenVoidForm>::value;

-  const static bool hasVoid =
+  constexpr static bool hasVoid =
      !HasStepTaken<CallbackType, TypedForms<OptimizerType,
          FunctionType, MatType>::template StepTakenBoolForm>::value &&
      HasStepTaken<CallbackType, TypedForms<OptimizerType,
-         FunctionType, MatType>::template StepTakenVoidForm>::value;
+          FunctionType, MatType>::template StepTakenVoidForm>::value;

-  const static bool hasNone =
+  constexpr static bool hasNone =
      !HasStepTaken<CallbackType, TypedForms<OptimizerType,
          FunctionType, MatType>::template StepTakenBoolForm>::value &&
      !HasStepTaken<CallbackType, TypedForms<OptimizerType,
-         FunctionType, MatType>::template StepTakenVoidForm>::value;
+          FunctionType, MatType>::template StepTakenVoidForm>::value;
 };

+//! A utility struct for Typed Forms required in
+//! callbacks for MultiObjective Optimizers.
+template<typename OptimizerType,
+         typename FunctionType,
+         typename MatType,
+         typename ObjectivesVecType,
+         typename IndicesType,
+         typename GradType = MatType>
+struct MOOTypedForms
+{
+  //! This is the form of a bool GenerationalStepTaken() for MOO callback method.
+  template<typename CallbackType>
+  using GenerationalStepTakenBoolForm =
+      bool(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const ObjectivesVecType&,
+                            const IndicesType&);
+
+  //! This is the form of a void StepTaken() for MOO callback method.
+  template<typename CallbackType>
+  using GenerationalStepTakenVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const ObjectivesVecType&,
+                            const IndicesType&);
+};
+
+//! Utility struct, check if either void StepTaken() or bool StepTaken() exists.
+//! Specialization for Multiobjective case.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename ObjectivesVecType,
+         typename IndicesType,
+         typename MatType>
+ struct HasGenerationalStepTakenSignature
+{
+  constexpr static bool hasBool =
+    HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenBoolForm>::value &&
+    !HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenVoidForm>::value;
+
+  constexpr static bool hasVoid =
+    !HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenBoolForm>::value &&
+    HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenVoidForm>::value;
+
+  constexpr static bool hasNone =
+    !HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenBoolForm>::value &&
+    !HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenVoidForm>::value;
+};
 } // namespace traits
 } // namespace callbacks
 } // namespace ens
@@ -1,16 +1,16 @@
 /**
- * @file scd.hpp
+ * @file cd.hpp
 * @author Shikhar Bhardwaj
 *
- * Stochastic Coordinate Descent (SCD).
+ * Coordinate Descent (CD).
 *
 * ensmallen is free software; you may redistribute it and/or modify it under
 * the terms of the 3-clause BSD license.  You should have received a copy of
 * the 3-clause BSD license along with ensmallen.  If not, see
 * http://www.opensource.org/licenses/BSD-3-Clause for more information.
 */
-#ifndef ENSMALLEN_SCD_SCD_HPP
-#define ENSMALLEN_SCD_SCD_HPP
+#ifndef ENSMALLEN_CD_CD_HPP
+#define ENSMALLEN_CD_CD_HPP

 #include "descent_policies/cyclic_descent.hpp"
 #include "descent_policies/random_descent.hpp"
@@ -42,7 +42,7 @@ namespace ens {
 * }
 * @endcode
 *
- * SCD can optimize partially differentiable functions.  For more details, see
+ * CD can optimize partially differentiable functions.  For more details, see
 * the documentation on function types included with this distribution or on the
 * ensmallen website.
 *
@@ -50,11 +50,11 @@ namespace ens {
 *     coordinate for descent is selected.
 */
 template <typename DescentPolicyType = RandomDescent>
-class SCD
+class CD
 {
 public:
  /**
-   * Construct the SCD optimizer with the given function and parameters. The
+   * Construct the CD optimizer with the given function and parameters. The
   * default value here are not necessarily good for every problem, so it is
   * suggested that the values used are tailored for the task at hand. The
   * maximum number of iterations refers to the maximum number of "descents"
@@ -70,11 +70,11 @@ class SCD
   * @param descentPolicy The policy to use for picking up the coordinate to
   *    descend on.
   */
-  SCD(const double stepSize = 0.01,
-      const size_t maxIterations = 100000,
-      const double tolerance = 1e-5,
-      const size_t updateInterval = 1e3,
-      const DescentPolicyType descentPolicy = DescentPolicyType());
+  CD(const double stepSize = 0.01,
+     const size_t maxIterations = 100000,
+     const double tolerance = 1e-5,
+     const size_t updateInterval = 1e3,
+     const DescentPolicyType descentPolicy = DescentPolicyType());

  /**
   * Optimize the given function using stochastic coordinate descent. The
@@ -158,6 +158,24 @@ class SCD
 } // namespace ens

 // Include implementation.
-#include "scd_impl.hpp"
+#include "cd_impl.hpp"
+
+namespace ens {
+
+/**
+ * Backwards-compatibility alias; this can be removed after ensmallen 3.10.0.
+ * The history here is that CD was originally named SCD, but that is an
+ * inaccurate name because this is not a stochastic technique; thus, it was
+ * renamed SCD.
+ */
+template<typename DescentPolicyType = RandomDescent>
+using SCD = CD<DescentPolicyType>;
+
+// Convenience typedefs.
+using RandomCD = CD<RandomDescent>;
+using GreedyCD = CD<GreedyDescent>;
+using CyclicCD = CD<CyclicDescent>;
+
+} // namespace ens

 #endif
@@ -1,26 +1,26 @@
 /**
- * @file scd_impl.hpp
+ * @file cd_impl.hpp
 * @author Shikhar Bhardwaj
 *
- * Implementation of stochastic coordinate descent.
+ * Implementation of coordinate descent.
 *
 * ensmallen is free software; you may redistribute it and/or modify it under
 * the terms of the 3-clause BSD license.  You should have received a copy of
 * the 3-clause BSD license along with ensmallen.  If not, see
 * http://www.opensource.org/licenses/BSD-3-Clause for more information.
 */
-#ifndef ENSMALLEN_SCD_SCD_IMPL_HPP
-#define ENSMALLEN_SCD_SCD_IMPL_HPP
+#ifndef ENSMALLEN_CD_CD_IMPL_HPP
+#define ENSMALLEN_CD_CD_IMPL_HPP

 // In case it hasn't been included yet.
-#include "scd.hpp"
+#include "cd.hpp"

 #include <ensmallen_bits/function.hpp>

 namespace ens {

 template <typename DescentPolicyType>
-SCD<DescentPolicyType>::SCD(
+CD<DescentPolicyType>::CD(
    const double stepSize,
    const size_t maxIterations,
    const double tolerance,
@@ -41,7 +41,7 @@ template <typename ResolvableFunctionType,
          typename... CallbackTypes>
 typename std::enable_if<IsArmaType<GradType>::value,
 typename MatType::elem_type>::type
-SCD<DescentPolicyType>::Optimize(
+CD<DescentPolicyType>::Optimize(
    ResolvableFunctionType& function,
    MatType& iterateIn,
    CallbackTypes&&... callbacks)
@@ -67,8 +67,7 @@ SCD<DescentPolicyType>::Optimize(
  bool terminate = false;

  // Start iterating.
-  terminate |= Callback::BeginOptimization(*this, function, iterate,
-      callbacks...);
+  Callback::BeginOptimization(*this, function, iterate, callbacks...);
  for (size_t i = 1; i != maxIterations && !terminate; ++i)
  {
    // Get the coordinate to descend on.
@@ -81,6 +80,8 @@ SCD<DescentPolicyType>::Optimize(

    terminate |= Callback::Gradient(*this, function, iterate, overallObjective,
        gradient, callbacks...);
+    if (terminate)
+      break;

    // Update the decision variable with the partial gradient.
    iterate.col(featureIdx) -= stepSize * gradient.col(featureIdx);
@@ -94,12 +95,12 @@ SCD<DescentPolicyType>::Optimize(
          overallObjective, callbacks...);

      // Output current objective function.
-      Info << "SCD: iteration " << i << ", objective " << overallObjective
+      Info << "CD: iteration " << i << ", objective " << overallObjective
          << "." << std::endl;

      if (std::isnan(overallObjective) || std::isinf(overallObjective))
      {
-        Warn << "SCD: converged to " << overallObjective << "; terminating"
+        Warn << "CD: converged to " << overallObjective << "; terminating"
            << " with failure.  Try a smaller step size?" << std::endl;

        Callback::EndOptimization(*this, function, iterate, callbacks...);
@@ -108,7 +109,7 @@ SCD<DescentPolicyType>::Optimize(

      if (std::abs(lastObjective - overallObjective) < tolerance)
      {
-        Info << "SCD: minimized within tolerance " << tolerance << "; "
+        Info << "CD: minimized within tolerance " << tolerance << "; "
            << "terminating optimization." << std::endl;

        Callback::EndOptimization(*this, function, iterate, callbacks...);
@@ -119,12 +120,13 @@ SCD<DescentPolicyType>::Optimize(
    }
  }

-  Info << "SCD: maximum iterations (" << maxIterations << ") reached; "
+  Info << "CD: maximum iterations (" << maxIterations << ") reached; "
      << "terminating optimization." << std::endl;

-  // Calculate and return final objective.
+  // Calculate and return final objective.  No need to pay attention to the
+  // result of the callback.
  const ElemType objective = function.Evaluate(iterate);
-  Callback::Evaluate(*this, function, iterate, objective, callbacks...);
+  (void) Callback::Evaluate(*this, function, iterate, objective, callbacks...);

  Callback::EndOptimization(*this, function, iterate, callbacks...);
  return objective;
@@ -0,0 +1,219 @@
+/**
+ * @file active_cmaes.hpp
+ * @author Marcus Edel
+ * @author Suvarsha Chennareddy
+ *
+ * Definition of the Active Covariance Matrix Adaptation Evolution Strategy 
+ * as proposed by G.A Jastrebski and D.V Arnold in "Improving Evolution 
+ * Strategies through Active Covariance Matrix Adaptation".
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CMAES_ACTIVE_CMAES_HPP
+#define ENSMALLEN_CMAES_ACTIVE_CMAES_HPP
+
+#include "full_selection.hpp"
+#include "random_selection.hpp"
+#include "transformation_policies/empty_transformation.hpp"
+#include "transformation_policies/boundary_box_constraint.hpp"
+
+namespace ens {
+
+/**
+ * Active CMA-ES is a variant of the stochastic search algorithm
+ * CMA-ES - Covariance Matrix Adaptation Evolution Strategy.
+ * Active CMA-ES actively reduces the uncertainty in unfavourable directions by
+ * exploiting the information about bad mutations in the covariance matrix 
+ * update step. This isn't for the purpose of accelerating progress, but 
+ * instead for speeding up the adaptation of the covariance matrix (which, in 
+ * turn, will lead to faster progress).
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @INPROCEEDINGS{1688662,
+ *   author={Jastrebski, G.A. and Arnold, D.V.},
+ *   booktitle={2006 IEEE International Conference on Evolutionary 
+                Computation},
+ *   title={Improving Evolution Strategies through Active Covariance 
+            Matrix Adaptation},
+ *   year={2006},
+ *   volume={},
+ *   number={},
+ *   pages={2814-2821},
+ *   doi={10.1109/CEC.2006.1688662}}
+ * @endcode
+ *
+ * Active CMA-ES can optimize separable functions.  For more details, see the
+ * documentation on function types included with this distribution or on the
+ * ensmallen website.
+ *
+ * @tparam SelectionPolicy The selection strategy used for the evaluation step.
+ * @tparam TransformationPolicy The transformation strategy used to 
+ *       map decision variables to the desired domain during fitness evaluation
+ *       and termination. Use EmptyTransformation if the domain isn't bounded.
+ */
+template<typename SelectionPolicyType = FullSelection,
+         typename TransformationPolicyType = EmptyTransformation<>>
+class ActiveCMAES
+{
+ public:
+  /**
+   * Construct the Active CMA-ES optimizer with the given function and parameters. The
+   * defaults here are not necessarily good for the given problem, so it is
+   * suggested that the values used be tailored to the task at hand.  The
+   * maximum number of iterations refers to the maximum number of points that
+   * are processed (i.e., one iteration equals one point; one iteration does not
+   * equal one pass over the dataset).
+   *
+   * @param lambda The population size (0 use the default size).
+   * @param transformationPolicy Instantiated transformation policy used to 
+   *     map the coordinates to the desired domain.
+   * @param batchSize Batch size to use for the objective calculation.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *     limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param selectionPolicy Instantiated selection policy used to calculate the
+   *     objective.
+   * @param stepSize Starting sigma/step size (will be modified).
+   */
+  ActiveCMAES(
+      const size_t lambda = 0,
+      const TransformationPolicyType& 
+          transformationPolicy = TransformationPolicyType(),
+      const size_t batchSize = 32,
+      const size_t maxIterations = 1000,
+      const double tolerance = 1e-5,
+      const SelectionPolicyType& selectionPolicy = SelectionPolicyType(),
+      double stepSize = 0);
+
+  /**
+   * Construct the Active CMA-ES optimizer with the given function and parameters 
+   * (including lower and upper bounds). The defaults here are not necessarily 
+   * good for the given problem, so it is suggested that the values used be 
+   * tailored to the task at hand.  The maximum number of iterations refers to 
+   * the maximum number of points that are processed (i.e., one iteration 
+   * equals one point; one iteration does not equal one pass over the dataset). 
+   *
+   * @param lambda The population size(0 use the default size).
+   * @param lowerBound Lower bound of decision variables.
+   * @param upperBound Upper bound of decision variables.
+   * @param batchSize Batch size to use for the objective calculation.
+   * @param maxIterations Maximum number of iterations allowed(0 means no
+      limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param selectionPolicy Instantiated selection policy used to calculate the
+   * objective.
+   * @param stepSize Starting sigma/step size (will be modified).
+   */
+  ActiveCMAES(
+      const size_t lambda = 0,
+      const double lowerBound = -10,
+      const double upperBound = 10,
+      const size_t batchSize = 32,
+      const size_t maxIterations = 1000,
+      const double tolerance = 1e-5,
+      const SelectionPolicyType& selectionPolicy = SelectionPolicyType(),
+      double stepSize = 0);
+
+  /**
+   * Optimize the given function using Active CMA-ES. The given starting point will be
+   * modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+      typename MatType,
+      typename... CallbackTypes>
+      typename MatType::elem_type Optimize(
+          SeparableFunctionType& function,
+          MatType& iterate,
+          CallbackTypes&&... callbacks);
+
+  //! Get the population size.
+  size_t PopulationSize() const { return lambda; }
+  //! Modify the population size.
+  size_t& PopulationSize() { return lambda; }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return batchSize; }
+  //! Modify the batch size.
+  size_t& BatchSize() { return batchSize; }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return maxIterations; }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return maxIterations; }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return tolerance; }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return tolerance; }
+
+  //! Get the selection policy.
+  const SelectionPolicyType& SelectionPolicy() const { return selectionPolicy; }
+  //! Modify the selection policy.
+  SelectionPolicyType& SelectionPolicy() { return selectionPolicy; }
+
+  //! Get the transformation policy.
+  const TransformationPolicyType& TransformationPolicy() const
+  { return transformationPolicy; }
+  //! Modify the transformation policy.
+  TransformationPolicyType& TransformationPolicy() 
+  { return transformationPolicy; }
+
+  //! Get the step size.
+  double StepSize() const
+  { return stepSize; }
+  //! Modify the step size.
+  double& StepSize()
+  { return stepSize; }
+
+ private:
+  //! Population size.
+  size_t lambda;
+
+  //! The batch size for processing.
+  size_t batchSize;
+
+  //! The maximum number of allowed iterations.
+  size_t maxIterations;
+
+  //! The tolerance for termination.
+  double tolerance;
+
+  //! The selection policy used to calculate the objective.
+  SelectionPolicyType selectionPolicy;
+
+  //! The transformationPolicy used to map coordinates to the suitable domain
+  //! while evaluating fitness. This mapping is also done after optimization 
+  //! has completed.
+  TransformationPolicyType transformationPolicy;
+
+  //! The step size.
+  double stepSize;
+};
+
+/**
+ * Convenient typedef for Active CMAES approximation.
+ */
+template<typename TransformationPolicyType = EmptyTransformation<>,
+         typename SelectionPolicyType = RandomSelection>
+using ApproxActiveCMAES = ActiveCMAES<SelectionPolicyType, TransformationPolicyType>;
+
+} // namespace ens
+
+// Include implementation.
+#include "active_cmaes_impl.hpp"
+
+#endif
@@ -0,0 +1,369 @@
+/**
+ * @file active_cmaes_impl.hpp
+ * @author Marcus Edel
+ * @author Suvarsha Chennareddy
+ *
+ * Implementation of the Active Covariance Matrix Adaptation Evolution Strategy
+ * as proposed by G.A Jastrebski and D.V Arnold in "Improving Evolution
+ * Strategies through Active Covariance Matrix Adaptation".
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CMAES_ACTIVE_CMAES_IMPL_HPP
+#define ENSMALLEN_CMAES_ACTIVE_CMAES_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "active_cmaes.hpp"
+
+#include "not_empty_transformation.hpp"
+#include <ensmallen_bits/function.hpp>
+
+namespace ens {
+
+template<typename SelectionPolicyType, typename TransformationPolicyType>
+ActiveCMAES<SelectionPolicyType, TransformationPolicyType>::ActiveCMAES(
+                                  const size_t lambda,
+                                  const TransformationPolicyType& 
+                                        transformationPolicy,
+                                  const size_t batchSize,
+                                  const size_t maxIterations,
+                                  const double tolerance,
+                                  const SelectionPolicyType& selectionPolicy,
+                                  double stepSizeIn) :
+    lambda(lambda),
+    batchSize(batchSize),
+    maxIterations(maxIterations),
+    tolerance(tolerance),
+    selectionPolicy(selectionPolicy),
+    transformationPolicy(transformationPolicy),
+    stepSize(stepSizeIn)
+{ /* Nothing to do. */ }
+
+template<typename SelectionPolicyType, typename TransformationPolicyType>
+ActiveCMAES<SelectionPolicyType, TransformationPolicyType>::ActiveCMAES(
+                                  const size_t lambda,
+                                  const double lowerBound,
+                                  const double upperBound,
+                                  const size_t batchSize,
+                                  const size_t maxIterations,
+                                  const double tolerance,
+                                  const SelectionPolicyType& selectionPolicy,
+                                  double stepSizeIn) :
+    lambda(lambda),
+    batchSize(batchSize),
+    maxIterations(maxIterations),
+    tolerance(tolerance),
+    selectionPolicy(selectionPolicy),
+    stepSize(stepSizeIn)
+{
+  Warn << "This is a deprecated constructor and will be removed in a "
+    "future version of ensmallen" << std::endl;
+  NotEmptyTransformation<TransformationPolicyType, EmptyTransformation<>> d;
+  d.Assign(transformationPolicy, lowerBound, upperBound);
+}
+
+//! Optimize the function (minimize).
+template<typename SelectionPolicyType, typename TransformationPolicyType>
+template<typename SeparableFunctionType,
+         typename MatType,
+         typename... CallbackTypes>
+typename MatType::elem_type ActiveCMAES<SelectionPolicyType, 
+  TransformationPolicyType>::Optimize(
+    SeparableFunctionType& function,
+    MatType& iterateIn,
+    CallbackTypes&&... callbacks)
+{
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
+  // Make sure that we have the methods that we need.  Long name...
+  traits::CheckArbitrarySeparableFunctionTypeAPI<
+      SeparableFunctionType, BaseMatType>();
+  RequireDenseFloatingPointType<BaseMatType>();
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  // Find the number of functions to use.
+  const size_t numFunctions = function.NumFunctions();
+
+  // Population size.
+  if (lambda == 0)
+    lambda = (4 + std::round(3 * std::log(iterate.n_elem))) * 10;
+
+  // Parent number.
+  const size_t mu = std::round(lambda / 4);
+
+  // Recombination weight (w = 1 / (parent number)).
+  const ElemType w = 1.0 / mu;
+
+  // Number of effective solutions.
+  const ElemType muEffective = mu;
+
+  // Step size control parameters.
+  BaseMatType sigma(2, 1); // sigma is vector-shaped.
+  if (stepSize == 0) 
+    sigma(0) = transformationPolicy.InitialStepSize();
+  else 
+    sigma(0) = stepSize;
+
+  const ElemType cs = 4.0 / (iterate.n_elem + 4);
+  const ElemType ds = 1 + cs;
+  const ElemType enn = std::sqrt(iterate.n_elem) * (1.0 - 1.0 /
+      (4.0 * iterate.n_elem) + 1.0 / (21 * std::pow(iterate.n_elem, 2)));
+
+  // Covariance update parameters. Cumulation for distribution.
+  const ElemType cc = cs;
+  const ElemType ccov = 2.0 / std::pow((iterate.n_elem + std::sqrt(2)), 2);
+  const ElemType beta = (4.0 * mu - 2.0) / (std::pow((iterate.n_elem + 12), 2) 
+      + 4 * mu);
+
+  std::vector<BaseMatType> mPosition(2, BaseMatType(iterate.n_rows,
+      iterate.n_cols));
+  mPosition[0] = iterate;
+
+  BaseMatType step(iterate.n_rows, iterate.n_cols);
+  step.zeros();
+
+  BaseMatType transformedIterate = transformationPolicy.Transform(iterate);
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
+  // Calculate the first objective function.
+  ElemType currentObjective = 0;
+  for (size_t f = 0; f < numFunctions; f += batchSize)
+  {
+    const size_t effectiveBatchSize = std::min(batchSize, numFunctions - f);
+    const ElemType objective = function.Evaluate(transformedIterate, f,
+        effectiveBatchSize);
+    currentObjective += objective;
+
+    terminate |= Callback::Evaluate(*this, function, transformedIterate,
+        objective, callbacks...);
+  }
+
+  ElemType overallObjective = currentObjective;
+  ElemType lastObjective = std::numeric_limits<ElemType>::max();
+
+  // Population parameters.
+  std::vector<BaseMatType> pStep(lambda, BaseMatType(iterate.n_rows,
+      iterate.n_cols));
+  std::vector<BaseMatType> pPosition(lambda, BaseMatType(iterate.n_rows,
+      iterate.n_cols));
+  BaseMatType pObjective(lambda, 1); // pObjective is vector-shaped.
+  std::vector<BaseMatType> ps(2, BaseMatType(iterate.n_rows, iterate.n_cols));
+  ps[0].zeros();
+  ps[1].zeros();
+  std::vector<BaseMatType> pc = ps;
+  std::vector<BaseMatType> C(2, BaseMatType(iterate.n_elem, iterate.n_elem));
+  C[0].eye();
+
+  // Covariance matrix parameters.
+  arma::Col<ElemType> eigval;
+  BaseMatType eigvec;
+  BaseMatType eigvalZero(iterate.n_elem, 1); // eigvalZero is vector-shaped.
+  eigvalZero.zeros();
+
+  // The current visitation order (sorted by population objectives).
+  arma::uvec idx = arma::linspace<arma::uvec>(0, lambda - 1, lambda);
+
+  // Now iterate!
+  Callback::BeginOptimization(*this, function, transformedIterate,
+      callbacks...);
+
+  size_t idx0, idx1;
+
+  // The number of generations to wait after the minimum loss has
+  // been reached or no improvement has been made before terminating.
+  size_t patience = 10 + (30 * iterate.n_elem / lambda) + 1;
+  size_t steps = 0;
+
+  for (size_t i = 1; (i != maxIterations) && !terminate; ++i)
+  {
+    // To keep track of where we are.
+    idx0 = (i - 1) % 2;
+    idx1 = i % 2;
+
+    // Perform Cholesky decomposition. If the matrix is not positive definite,
+    // add a small value and try again.
+    BaseMatType covLower;
+    while (!arma::chol(covLower, C[idx0], "lower"))
+      C[idx0].diag() += std::numeric_limits<ElemType>::epsilon();
+
+    arma::eig_sym(eigval, eigvec, C[idx0]);
+
+    for (size_t j = 0; j < lambda; ++j)
+    {
+      if (iterate.n_rows > iterate.n_cols)
+      {
+        pStep[idx(j)] = covLower *
+          arma::randn<BaseMatType>(iterate.n_rows, iterate.n_cols);
+      }
+      else
+      {
+        pStep[idx(j)] = arma::randn<BaseMatType>(iterate.n_rows, iterate.n_cols)
+          * covLower.t();
+      }
+
+      pPosition[idx(j)] = mPosition[idx0] + sigma(idx0) * pStep[idx(j)];
+
+      // Calculate the objective function.
+      pObjective(idx(j)) = selectionPolicy.Select(function, batchSize,
+          transformationPolicy.Transform(pPosition[idx(j)]), terminate,
+          callbacks...);
+    }
+
+    // Sort population.
+    idx = arma::sort_index(pObjective);
+
+    step = w * pStep[idx(0)];
+    for (size_t j = 1; j < mu; ++j)
+      step += w * pStep[idx(j)];
+
+    mPosition[idx1] = mPosition[idx0] + sigma(idx0) * step;
+
+    // Calculate the objective function.
+    currentObjective = selectionPolicy.Select(function, batchSize,
+        transformationPolicy.Transform(mPosition[idx1]), terminate,
+        callbacks...);
+
+    // Update best parameters.
+    if (currentObjective < overallObjective)
+    {
+      overallObjective = currentObjective;
+      iterate = mPosition[idx1];
+
+      transformedIterate = transformationPolicy.Transform(iterate);
+      terminate |= Callback::StepTaken(*this, function,
+          transformedIterate, callbacks...);
+    }
+
+    // Update Step Size.
+    if (iterate.n_rows > iterate.n_cols)
+    {
+      ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt(
+        cs * (2 - cs) * muEffective) *
+        eigvec * diagmat(1 / eigval) * eigvec.t() * step;
+    }
+    else
+    {
+      ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt(
+          cs * (2 - cs) * muEffective) * step *
+          eigvec * diagmat(1 / eigval) * eigvec.t();
+    }
+
+    const ElemType psNorm = arma::norm(ps[idx1]);
+    sigma(idx1) = sigma(idx0) * std::exp(cs / ds * (psNorm / enn - 1));
+
+    if (std::isnan(sigma(idx1)) || sigma(idx1) > 1e14)
+    {
+      Warn << "The step size diverged to " << sigma(idx1) << "; "
+          << "terminating with failure.  Try a smaller step size?" << std::endl;
+
+      iterate = transformationPolicy.Transform(iterate);
+
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
+      return overallObjective;
+    }
+
+    pc[idx1] = (1 - cc) * pc[idx0] + std::sqrt(cc * (2 - cc) *
+        muEffective) * step;
+
+    if (iterate.n_rows > iterate.n_cols)
+    {
+      C[idx1] = (1 - ccov) * C[idx0] + ccov *
+          (pc[idx1] * pc[idx1].t());
+
+      for (size_t j = 0; j < mu; ++j)
+      {
+        C[idx1] = C[idx1] + beta * w *
+            pStep[idx(j)] * pStep[idx(j)].t();
+      }
+
+      for (size_t j = lambda - mu; j < lambda; ++j)
+      {
+        C[idx1] = C[idx1] - beta * w *
+            pStep[idx(j)] * pStep[idx(j)].t();
+      }
+    }
+    else
+    {
+      C[idx1] = (1 - ccov) * C[idx0] + ccov *
+          (pc[idx1].t() * pc[idx1]);
+
+      for (size_t j = 0; j < mu; ++j)
+      {
+        C[idx1] = C[idx1] + beta * w *
+            pStep[idx(j)].t() * pStep[idx(j)];
+      }
+
+      for (size_t j = lambda - mu; j < lambda; ++j)
+      {
+        C[idx1] = C[idx1] - beta * w *
+            pStep[idx(j)].t() * pStep[idx(j)];
+      }
+    }
+
+    arma::eig_sym(eigval, eigvec, C[idx1]);
+    const arma::uvec negativeEigval = arma::find(eigval < 0, 1);
+    if (!negativeEigval.is_empty())
+    {
+      if (negativeEigval(0) == 0)
+      {
+        C[idx1].zeros();
+      }
+      else
+      {
+        C[idx1] = eigvec.cols(0, negativeEigval(0) - 1) *
+            arma::diagmat(eigval.subvec(0, negativeEigval(0) - 1)) *
+            eigvec.cols(0, negativeEigval(0) - 1).t();
+      }
+    }
+
+    // Output current objective function.
+    Info << "Active CMA-ES: iteration " << i << ", objective " << overallObjective
+        << "." << std::endl;
+
+    if (std::isnan(overallObjective) || std::isinf(overallObjective))
+    {
+      Warn << "Active CMA-ES: converged to " << overallObjective << "; "
+          << "terminating with failure.  Try a smaller step size?" << std::endl;
+
+      iterate = transformationPolicy.Transform(iterate);
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
+      return overallObjective;
+    }
+
+    if (std::abs(lastObjective - overallObjective) < tolerance)
+    {
+      if (steps > patience)
+      {
+        Info << "Active CMA-ES: minimized within tolerance " << tolerance << "; "
+            << "terminating optimization." << std::endl;
+
+        iterate = transformationPolicy.Transform(iterate);
+        Callback::EndOptimization(*this, function, iterate, callbacks...);
+        return overallObjective;
+      }
+    }
+    else
+    {
+      steps = 0;
+    }
+
+    steps++;
+    lastObjective = overallObjective;
+  }
+
+  iterate = transformationPolicy.Transform(iterate);
+  Callback::EndOptimization(*this, function, iterate, callbacks...);
+  return overallObjective;
+}
+
+} // namespace ens
+
+#endif
@@ -17,6 +17,8 @@

 #include "full_selection.hpp"
 #include "random_selection.hpp"
+#include "transformation_policies/empty_transformation.hpp"
+#include "transformation_policies/boundary_box_constraint.hpp"

 namespace ens {

@@ -46,8 +48,12 @@ namespace ens {
 * ensmallen website.
 *
 * @tparam SelectionPolicy The selection strategy used for the evaluation step.
+ * @tparam TransformationPolicy The transformation strategy used to 
+ *       map decision variables to the desired domain during fitness evaluation
+ *       and termination. Use EmptyTransformation if the domain isn't bounded.
 */
-template<typename SelectionPolicyType = FullSelection>
+template<typename SelectionPolicyType = FullSelection,
+         typename TransformationPolicyType = EmptyTransformation<>>
 class CMAES
 {
 public:
@@ -60,14 +66,43 @@ class CMAES
   * equal one pass over the dataset).
   *
   * @param lambda The population size (0 use the default size).
-   * @param lowerBound Lower bound of decision variables.
-   * @param upperBound Upper bound of decision variables.
+   * @param transformationPolicy Instantiated transformation policy used to 
+   *     map the coordinates to the desired domain.
   * @param batchSize Batch size to use for the objective calculation.
   * @param maxIterations Maximum number of iterations allowed (0 means no
   *     limit).
   * @param tolerance Maximum absolute tolerance to terminate algorithm.
   * @param selectionPolicy Instantiated selection policy used to calculate the
   *     objective.
+   * @param stepSize Starting sigma/step size (will be modified).
+   */
+  CMAES(const size_t lambda = 0,
+        const TransformationPolicyType& 
+              transformationPolicy = TransformationPolicyType(),
+        const size_t batchSize = 32,
+        const size_t maxIterations = 1000,
+        const double tolerance = 1e-5,
+        const SelectionPolicyType& selectionPolicy = SelectionPolicyType(),
+        double stepSize = 0);
+
+  /**
+   * Construct the CMA-ES optimizer with the given function and parameters 
+   * (including lower and upper bounds). The defaults here are not necessarily 
+   * good for the given problem, so it is suggested that the values used be 
+   * tailored to the task at hand.  The maximum number of iterations refers to 
+   * the maximum number of points that are processed (i.e., one iteration 
+   * equals one point; one iteration does not equal one pass over the dataset).
+   *
+   * @param lambda The population size(0 use the default size).
+   * @param lowerBound Lower bound of decision variables.
+   * @param upperBound Upper bound of decision variables.
+   * @param batchSize Batch size to use for the objective calculation.
+   * @param maxIterations Maximum number of iterations allowed(0 means no
+      limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param selectionPolicy Instantiated selection policy used to calculate the
+   * objective.
+   * @param stepSize Starting sigma/step size (will be modified).
   */
  CMAES(const size_t lambda = 0,
        const double lowerBound = -10,
@@ -75,7 +110,8 @@ class CMAES
        const size_t batchSize = 32,
        const size_t maxIterations = 1000,
        const double tolerance = 1e-5,
-        const SelectionPolicyType& selectionPolicy = SelectionPolicyType());
+        const SelectionPolicyType& selectionPolicy = SelectionPolicyType(),
+        double stepSize = 0);

  /**
   * Optimize the given function using CMA-ES. The given starting point will be
@@ -91,27 +127,17 @@ class CMAES
   * @return Objective value of the final point.
   */
  template<typename SeparableFunctionType,
-           typename MatType,
-           typename... CallbackTypes>
-  typename MatType::elem_type Optimize(SeparableFunctionType& function,
-                                       MatType& iterate,
-                                       CallbackTypes&&... callbacks);
+      typename MatType,
+      typename... CallbackTypes>
+      typename MatType::elem_type Optimize(SeparableFunctionType& function,
+          MatType& iterate,
+          CallbackTypes&&... callbacks);

-  //! Get the step size.
+  //! Get the population size.
  size_t PopulationSize() const { return lambda; }
-  //! Modify the step size.
+  //! Modify the population size.
  size_t& PopulationSize() { return lambda; }

-  //! Get the lower bound of decision variables.
-  double LowerBound() const { return lowerBound; }
-  //! Modify the lower bound of decision variables.
-  double& LowerBound() { return lowerBound; }
-
-  //! Get the upper bound of decision variables
-  double UpperBound() const { return upperBound; }
-  //! Modify the upper bound of decision variables
-  double& UpperBound() { return upperBound; }
-
  //! Get the batch size.
  size_t BatchSize() const { return batchSize; }
  //! Modify the batch size.
@@ -132,16 +158,27 @@ class CMAES
  //! Modify the selection policy.
  SelectionPolicyType& SelectionPolicy() { return selectionPolicy; }

+  //! Get the transformation policy.
+  const TransformationPolicyType& TransformationPolicy() const
+  { return transformationPolicy; }
+  //! Modify the transformation policy.
+  TransformationPolicyType& TransformationPolicy() 
+  { return transformationPolicy; }
+
+  //! Get the step size.
+  double StepSize() const
+  { return stepSize; }
+  //! Modify the step size.
+  double& StepSize()
+  { return stepSize; }
+
+  //! Get the total number of function evaluations.
+  size_t FunctionEvaluations() const  { return functionEvaluations; }
+
 private:
  //! Population size.
  size_t lambda;

-  //! Lower bound of decision variables.
-  double lowerBound;
-
-  //! Upper bound of decision variables
-  double upperBound;
-
  //! The batch size for processing.
  size_t batchSize;

@@ -153,13 +190,25 @@ class CMAES

  //! The selection policy used to calculate the objective.
  SelectionPolicyType selectionPolicy;
+
+  //! The transformationPolicy used to map coordinates to the suitable domain
+  //! while evaluating fitness. This mapping is also done after optimization 
+  //! has completed.
+  TransformationPolicyType transformationPolicy;
+
+  //! The step size.
+  double stepSize;
+
+  //! Counter for the number of function evaluations.
+  size_t functionEvaluations = 0;
 };

 /**
 * Convenient typedef for CMAES approximation.
 */
-template<typename SelectionPolicyType = RandomSelection>
-using ApproxCMAES = CMAES<SelectionPolicyType>;
+template<typename TransformationPolicyType = EmptyTransformation<>,
+         typename SelectionPolicyType = RandomSelection>
+using ApproxCMAES = CMAES<SelectionPolicyType, TransformationPolicyType>;

 } // namespace ens

@@ -18,33 +18,59 @@
 // In case it hasn't been included yet.
 #include "cmaes.hpp"

+#include "not_empty_transformation.hpp"
 #include <ensmallen_bits/function.hpp>

 namespace ens {

-template<typename SelectionPolicyType>
-CMAES<SelectionPolicyType>::CMAES(const size_t lambda,
+template<typename SelectionPolicyType, typename TransformationPolicyType>
+CMAES<SelectionPolicyType, TransformationPolicyType>::CMAES(const size_t lambda,
+                                  const TransformationPolicyType& 
+                                        transformationPolicy,
+                                  const size_t batchSize,
+                                  const size_t maxIterations,
+                                  const double tolerance,
+                                  const SelectionPolicyType& selectionPolicy,
+                                  double stepSizeIn) :
+    lambda(lambda),
+    batchSize(batchSize),
+    maxIterations(maxIterations),
+    tolerance(tolerance),
+    selectionPolicy(selectionPolicy),
+    transformationPolicy(transformationPolicy),
+    stepSize(stepSizeIn)
+{ /* Nothing to do. */ }
+
+template<typename SelectionPolicyType, typename TransformationPolicyType>
+CMAES<SelectionPolicyType, TransformationPolicyType>::CMAES(const size_t lambda,
                                  const double lowerBound,
                                  const double upperBound,
                                  const size_t batchSize,
                                  const size_t maxIterations,
                                  const double tolerance,
-                                  const SelectionPolicyType& selectionPolicy) :
+                                  const SelectionPolicyType& selectionPolicy,
+                                  double stepSizeIn) :
    lambda(lambda),
-    lowerBound(lowerBound),
-    upperBound(upperBound),
    batchSize(batchSize),
    maxIterations(maxIterations),
    tolerance(tolerance),
-    selectionPolicy(selectionPolicy)
-{ /* Nothing to do. */ }
+    selectionPolicy(selectionPolicy),
+    stepSize(stepSizeIn)
+{
+  Warn << "This is a deprecated constructor and will be removed in a "
+    "future version of ensmallen" << std::endl;
+  NotEmptyTransformation<TransformationPolicyType, EmptyTransformation<>> d;
+  d.Assign(transformationPolicy, lowerBound, upperBound);
+}
+

 //! Optimize the function (minimize).
-template<typename SelectionPolicyType>
+template<typename SelectionPolicyType, typename TransformationPolicyType>
 template<typename SeparableFunctionType,
         typename MatType,
         typename... CallbackTypes>
-typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
+typename MatType::elem_type CMAES<SelectionPolicyType, 
+  TransformationPolicyType>::Optimize(
    SeparableFunctionType& function,
    MatType& iterateIn,
    CallbackTypes&&... callbacks)
@@ -78,7 +104,11 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(

  // Step size control parameters.
  BaseMatType sigma(2, 1); // sigma is vector-shaped.
-  sigma(0) = 0.3 * (upperBound - lowerBound);
+  if (stepSize == 0) 
+    sigma(0) = transformationPolicy.InitialStepSize();
+  else 
+    sigma(0) = stepSize;
+
  const double cs = (muEffective + 2) / (iterate.n_elem + muEffective + 5);
  const double ds = 1 + cs + 2 * std::max(std::sqrt((muEffective - 1) /
      (iterate.n_elem + 1)) - 1, 0.0);
@@ -99,24 +129,29 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(

  std::vector<BaseMatType> mPosition(2, BaseMatType(iterate.n_rows,
      iterate.n_cols));
-  mPosition[0] = lowerBound + arma::randu<BaseMatType>(
-      iterate.n_rows, iterate.n_cols) * (upperBound - lowerBound);
+  mPosition[0] = iterate;

  BaseMatType step(iterate.n_rows, iterate.n_cols);
  step.zeros();

+  BaseMatType transformedIterate = transformationPolicy.Transform(iterate);
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
  // Calculate the first objective function.
  ElemType currentObjective = 0;
  for (size_t f = 0; f < numFunctions; f += batchSize)
  {
    const size_t effectiveBatchSize = std::min(batchSize, numFunctions - f);
-    const ElemType objective = function.Evaluate(mPosition[0], f,
+    const ElemType objective = function.Evaluate(transformedIterate, f,
        effectiveBatchSize);
    currentObjective += objective;

-    Callback::Evaluate(*this, function, mPosition[0], objective,
-        callbacks...);
+    terminate |= Callback::Evaluate(*this, function, transformedIterate,
+        objective, callbacks...);
  }
+  functionEvaluations += numFunctions;

  ElemType overallObjective = currentObjective;
  ElemType lastObjective = std::numeric_limits<ElemType>::max();
@@ -143,13 +178,16 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
  // The current visitation order (sorted by population objectives).
  arma::uvec idx = arma::linspace<arma::uvec>(0, lambda - 1, lambda);

-  // Controls early termination of the optimization process.
-  bool terminate = false;
-
  // Now iterate!
-  terminate |= Callback::BeginOptimization(*this, function, iterate,
+  Callback::BeginOptimization(*this, function, transformedIterate,
      callbacks...);
-  for (size_t i = 1; i < maxIterations && !terminate; ++i)
+
+  // The number of generations to wait after the minimum loss has
+  // been reached or no improvement has been made before terminating.
+  size_t patience = 10 + (30 * iterate.n_elem / lambda) + 1;
+  size_t steps = 0;
+
+  for (size_t i = 1; (i != maxIterations) && !terminate; ++i)
  {
    // To keep track of where we are.
    const size_t idx0 = (i - 1) % 2;
@@ -159,26 +197,29 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
    // add a small value and try again.
    BaseMatType covLower;
    while (!arma::chol(covLower, C[idx0], "lower"))
-      C[idx0].diag() += 1e-16;
+      C[idx0].diag() += std::numeric_limits<ElemType>::epsilon();
+
+    arma::eig_sym(eigval, eigvec, C[idx0]);

    for (size_t j = 0; j < lambda; ++j)
    {
      if (iterate.n_rows > iterate.n_cols)
      {
        pStep[idx(j)] = covLower *
-            arma::randn<BaseMatType>(iterate.n_rows, iterate.n_cols);
+          arma::randn<BaseMatType>(iterate.n_rows, iterate.n_cols);
      }
      else
      {
        pStep[idx(j)] = arma::randn<BaseMatType>(iterate.n_rows, iterate.n_cols)
-            * covLower;
+          * covLower.t();
      }

      pPosition[idx(j)] = mPosition[idx0] + sigma(idx0) * pStep[idx(j)];

      // Calculate the objective function.
      pObjective(idx(j)) = selectionPolicy.Select(function, batchSize,
-          pPosition[idx(j)], callbacks...);
+          transformationPolicy.Transform(pPosition[idx(j)]), terminate,
+          callbacks...);
    }

    // Sort population.
@@ -192,7 +233,10 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(

    // Calculate the objective function.
    currentObjective = selectionPolicy.Select(function, batchSize,
-        mPosition[idx1], callbacks...);
+        transformationPolicy.Transform(mPosition[idx1]), terminate,
+        callbacks...);
+
+    functionEvaluations += lambda; 

    // Update best parameters.
    if (currentObjective < overallObjective)
@@ -200,23 +244,38 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
      overallObjective = currentObjective;
      iterate = mPosition[idx1];

-      terminate |= Callback::StepTaken(*this, function, iterate, callbacks...);
+      transformedIterate = transformationPolicy.Transform(iterate);
+      terminate |= Callback::StepTaken(*this, function,
+        transformedIterate, callbacks...);
    }

    // Update Step Size.
    if (iterate.n_rows > iterate.n_cols)
    {
      ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt(
-          cs * (2 - cs) * muEffective) * covLower.t() * step;
+        cs * (2 - cs) * muEffective) *
+        eigvec * diagmat(1 / eigval) * eigvec.t() * step;
    }
    else
    {
      ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt(
-          cs * (2 - cs) * muEffective) * step * covLower.t();
+        cs * (2 - cs) * muEffective) * step *
+        eigvec * diagmat(1 / eigval) * eigvec.t();
    }

    const ElemType psNorm = arma::norm(ps[idx1]);
-    sigma(idx1) = sigma(idx0) * std::exp(cs / ds * ( psNorm / enn - 1));
+    sigma(idx1) = sigma(idx0) * std::exp(cs / ds * (psNorm / enn - 1));
+
+    if (std::isnan(sigma(idx1)) || sigma(idx1) > 1e14)
+    {
+      Warn << "The step size diverged to " << sigma(idx1) << "; "
+        << "terminating with failure.  Try a smaller step size?" << std::endl;
+
+      iterate = transformationPolicy.Transform(iterate);
+
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
+      return overallObjective;
+    }

    // Update covariance matrix.
    if ((psNorm / sqrt(1 - std::pow(1 - cs, 2 * i))) < h)
@@ -242,12 +301,12 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
      if (iterate.n_rows > iterate.n_cols)
      {
        C[idx1] = (1 - c1 - cmu) * C[idx0] + c1 * (pc[idx1] *
-            pc[idx1].t() + (cc * (2 - cc)) * C[idx0]);
+          pc[idx1].t() + (cc * (2 - cc)) * C[idx0]);
      }
      else
      {
        C[idx1] = (1 - c1 - cmu) * C[idx0] + c1 *
-            (pc[idx1].t() * pc[idx1] + (cc * (2 - cc)) * C[idx0]);
+          (pc[idx1].t() * pc[idx1] + (cc * (2 - cc)) * C[idx0]);
      }
    }

@@ -256,7 +315,7 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
      for (size_t j = 0; j < mu; ++j)
      {
        C[idx1] = C[idx1] + cmu * w(j) *
-            pStep[idx(j)] * pStep[idx(j)].t();
+          pStep[idx(j)] * pStep[idx(j)].t();
      }
    }
    else
@@ -264,7 +323,7 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
      for (size_t j = 0; j < mu; ++j)
      {
        C[idx1] = C[idx1] + cmu * w(j) *
-            pStep[idx(j)].t() * pStep[idx(j)];
+          pStep[idx(j)].t() * pStep[idx(j)];
      }
    }

@@ -279,36 +338,46 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
      else
      {
        C[idx1] = eigvec.cols(0, negativeEigval(0) - 1) *
-            arma::diagmat(eigval.subvec(0, negativeEigval(0) - 1)) *
-            eigvec.cols(0, negativeEigval(0) - 1).t();
+          arma::diagmat(eigval.subvec(0, negativeEigval(0) - 1)) *
+          eigvec.cols(0, negativeEigval(0) - 1).t();
      }
    }

    // Output current objective function.
    Info << "CMA-ES: iteration " << i << ", objective " << overallObjective
-        << "." << std::endl;
+      << "." << std::endl;

    if (std::isnan(overallObjective) || std::isinf(overallObjective))
    {
      Warn << "CMA-ES: converged to " << overallObjective << "; "
-          << "terminating with failure.  Try a smaller step size?" << std::endl;
+        << "terminating with failure.  Try a smaller step size?" << std::endl;

+      iterate = transformationPolicy.Transform(iterate);
      Callback::EndOptimization(*this, function, iterate, callbacks...);
      return overallObjective;
    }

    if (std::abs(lastObjective - overallObjective) < tolerance)
    {
-      Info << "CMA-ES: minimized within tolerance " << tolerance << "; "
+      if (steps > patience) {
+        Info << "CMA-ES: minimized within tolerance " << tolerance << "; "
          << "terminating optimization." << std::endl;

-      Callback::EndOptimization(*this, function, iterate, callbacks...);
-      return overallObjective;
+        iterate = transformationPolicy.Transform(iterate);
+        Callback::EndOptimization(*this, function, iterate, callbacks...);
+        return overallObjective;
+      }
    }
+    else {
+      steps = 0;
+    }
+
+    steps++;

    lastObjective = overallObjective;
  }

+  iterate = transformationPolicy.Transform(iterate);
  Callback::EndOptimization(*this, function, iterate, callbacks...);
  return overallObjective;
 }
@@ -26,6 +26,7 @@ class FullSelection
   * @tparam SeparableFunctionType Type of the function to be evaluated.
   * @param function Function to optimize.
   * @param batchSize Batch size to use for each step.
+   * @param terminate Whether optimization should be terminated after this call.
   * @param iterate starting point.
   */
  template<typename SeparableFunctionType,
@@ -34,6 +35,7 @@ class FullSelection
  double Select(SeparableFunctionType& function,
                const size_t batchSize,
                const MatType& iterate,
+                bool& terminate,
                CallbackTypes&... callbacks)
  {
    // Find the number of functions to use.
@@ -45,7 +47,8 @@ class FullSelection
      const size_t effectiveBatchSize = std::min(batchSize, numFunctions - f);
      objective += function.Evaluate(iterate, f, effectiveBatchSize);

-      Callback::Evaluate(*this, f, iterate, objective, callbacks...);
+      terminate |= Callback::Evaluate(*this, f, iterate, objective,
+          callbacks...);
    }

    return objective;
@@ -0,0 +1,42 @@
+/**
+ * @file not_empty_transformation.hpp
+ * @author Suvarsha Chennareddy
+ *
+ * Check whether TransformationPolicyType is EmptyTransformation.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CMAES_NOT_EMPTY_TRANSFORMATION
+#define ENSMALLEN_CMAES_NOT_EMPTY_TRANSFORMATION
+
+/**
+ * This partial specialization is used to throw an exception when the
+ * TransformationPolicyType is EmptyTransformation and call a constructor with
+ * parameters 'lowerBound' and 'upperBound' otherwise.  This shall be removed
+ * when the deprecated constructor is removed in the next major version of
+ * ensmallen.
+ */
+template<typename T1, typename T2>
+struct NotEmptyTransformation : std::true_type
+{
+  void Assign(T1& obj, double lowerBound, double upperBound)
+  {
+    obj = T1(lowerBound, upperBound);
+  }
+};
+
+template<template<typename...> class T, typename... A, typename... B>
+struct NotEmptyTransformation<T<A...>, T<B...>> : std::false_type
+{
+  void Assign(T<A...>& /* obj */,
+              double /* lowerBound */,
+              double /* upperBound */)
+  {
+    throw std::logic_error("TransformationPolicyType is EmptyTransformation");
+  }
+};
+
+#endif
@@ -0,0 +1,175 @@
+/**
+ * @file ipop_cmaes_impl.hpp
+ * @author Marcus Edel
+ * @author Benjami Parellada
+ *
+ * Definition of the IPOP Covariance Matrix Adaptation Evolution Strategy
+ * as proposed by A. Auger and N. Hansen in "A Restart CMA Evolution
+ * Strategy With Increasing Population Size" and BIPOP Covariance Matrix
+ * Adaptation Evolution Strategy as proposed by N. Hansen in "Benchmarking 
+ * a BI-population CMA-ES on the BBOB-2009 function testbed".
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CMAES_POP_CMAES_HPP
+#define ENSMALLEN_CMAES_POP_CMAES_HPP
+
+#include "cmaes.hpp"
+
+namespace ens {
+
+/**
+ * Population-based CMA-ES (POP-CMA-ES) that can operate as either IPOP-CMA-ES
+ * or BIPOP-CMA-ES based on a flag.
+ * 
+ * IPOP CMA-ES is a variant of the stochastic search algorithm
+ * CMA-ES - Covariance Matrix Adaptation Evolution Strategy.
+ * IPOP CMA-ES, also known as CMAES with increasing population size, 
+ * incorporates a restart strategy that involves gradually increasing
+ * the population size. This approach is specifically designed to 
+ * enhance the performance of CMA-ES on multi-modal functions.
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @INPROCEEDINGS{1554902,
+ *   author={Auger, A. and Hansen, N.},
+ *   booktitle={2005 IEEE Congress on Evolutionary Computation},
+ *   title={A restart CMA evolution strategy with increasing population size},
+ *   year={2005},
+ *   volume={2},
+ *   number={},
+ *   pages={1769-1776 Vol. 2},
+ *   doi={10.1109/CEC.2005.1554902}}
+ * @endcode
+ * 
+ * IPOP CMA-ES can optimize separable functions.  For more details, see the
+ * documentation on function types included with this distribution or on the
+ * ensmallen website.
+ * 
+ * BI-Population CMA-ES is a variant of the stochastic search algorithm
+ * CMA-ES - Covariance Matrix Adaptation Evolution Strategy.
+ * It implements a dual restart strategy with varying population sizes: one 
+ * increasing and one with smaller, varied sizes. This BI-population approach
+ * is designed to optimize performance on multi-modal function testbeds by 
+ * leveraging different exploration and exploitation dynamics.
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @inproceedings{hansen2009benchmarking,
+ *   title={Benchmarking a BI-population CMA-ES on the BBOB-2009 function testbed},
+ *   author={Hansen, Nikolaus},
+ *   booktitle={Proceedings of the 11th annual conference companion on genetic and evolutionary computation conference: late breaking papers},
+ *   pages={2389--2396},
+ *   year={2009}}
+ * @endcode
+ *
+ * BI-Population CMA-ES can efficiently handle separable, multimodal, and weak
+ * structure functions across various dimensions, as demonstrated in the 
+ * comprehensive results of the BBOB-2009 function testbed. The optimizer
+ * utilizes an interlaced multistart strategy to balance between broad 
+ * exploration and intensive exploitation, adjusting population sizes and 
+ * step-sizes dynamically.
+ */
+template<typename SelectionPolicyType = FullSelection,
+         typename TransformationPolicyType = EmptyTransformation<>,
+         bool UseBIPOPFlag = true>
+class POP_CMAES : public CMAES<SelectionPolicyType, TransformationPolicyType>
+{
+ public:
+  /**
+   * Construct the POP-CMA-ES optimizer with the given parameters.
+   * Other than the same CMA-ES parameters, it also adds the maximum number of 
+   * restarts, the increase in population factor, the maximum number of 
+   * evaluations, as well as a flag indicating to use BIPOP or not.
+   * The suggested values are not necessarily good for the given problem, so it
+   * is suggested that the values used be tailored to the task at hand. The
+   * maximum number of iterations refers to the maximum number of points that
+   * are processed (i.e., one iteration equals one point; one iteration does not
+   * equal one pass over the dataset).
+   * 
+   * @param lambda The initial population size (0 use the default size).
+   * @param transformationPolicy Instantiated transformation policy used to
+   *    map the coordinates to the desired domain.
+   * @param batchSize Batch size to use for the objective calculation.
+   * @param maxIterations Maximum number of iterations allowed.
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param selectionPolicy Instantiated selection policy used to calculate the
+   *     objective.
+   * @param stepSize Starting sigma/step size (will be modified).
+   * @param populationFactor The factor by which population increases
+   *     after each restart.
+   * @param maxRestarts Maximum number of restarts.
+   * @param maxFunctionEvaluations Maximum number of function evaluations.
+   */
+  POP_CMAES(const size_t lambda = 0,
+            const TransformationPolicyType& transformationPolicy = 
+                 TransformationPolicyType(),
+            const size_t batchSize = 32,
+            const size_t maxIterations = 1000,
+            const double tolerance = 1e-5,
+            const SelectionPolicyType& selectionPolicy = SelectionPolicyType(),
+            double stepSize = 0,
+            const size_t maxRestarts = 9,
+            const double populationFactor = 2,
+            const size_t maxFunctionEvaluations = 1e9);
+
+  /**
+   * Set POP-CMA-ES specific parameters.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks);
+
+  //! Get the population factor.
+  double PopulationFactor() const { return populationFactor; }
+  //! Modify the population factor.
+  double& PopulationFactor() { return populationFactor; }
+
+  //! Get the maximum number of restarts.
+  size_t MaxRestarts() const { return maxRestarts; }
+  //! Modify the maximum number of restarts.
+  size_t& MaxRestarts() { return maxRestarts; }
+
+  //! Get the maximum number of function evaluations.
+  size_t MaxFunctionEvaluations() const { return maxFunctionEvaluations; }
+  //! Modify the maximum number of function evaluations.
+  size_t& MaxFunctionEvaluations() { return maxFunctionEvaluations; }
+
+  //! Get the BIPOP mode flag.
+  static constexpr bool UseBIPOP() { return UseBIPOPFlag; }
+
+ private:
+  //! Population factor
+  double populationFactor;
+
+  //! Maximum number of restarts.
+  size_t maxRestarts;
+
+  //! Maximum number of function evaluations.
+  size_t maxFunctionEvaluations;
+
+};
+
+// Define IPOP_CMAES and BIPOP_CMAES using the POP_CMAES template
+template<typename SelectionPolicyType = FullSelection,
+         typename TransformationPolicyType = EmptyTransformation<>>
+using IPOP_CMAES = POP_CMAES<SelectionPolicyType, TransformationPolicyType, false>;
+
+template<typename SelectionPolicyType = FullSelection,
+         typename TransformationPolicyType = EmptyTransformation<>>
+using BIPOP_CMAES = POP_CMAES<SelectionPolicyType, TransformationPolicyType, true>;
+
+} // namespace ens
+
+// Include implementation.
+#include "pop_cmaes_impl.hpp"
+
+#endif
@@ -0,0 +1,163 @@
+/**
+ * @file ipop_cmaes_impl.hpp
+ * @author Marcus Edel
+ * @author Benjami Parellada
+ *
+ * Implementation of the IPOP Covariance Matrix Adaptation Evolution Strategy
+ * as proposed by A. Auger and N. Hansen in "A Restart CMA Evolution
+ * Strategy With Increasing Population Size" and BIPOP Covariance Matrix
+ * Adaptation Evolution Strategy as proposed by N. Hansen in "Benchmarking 
+ * a BI-population CMA-ES on the BBOB-2009 function testbed".
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CMAES_POP_CMAES_IMPL_HPP
+#define ENSMALLEN_CMAES_POP_CMAES_IMPL_HPP
+
+#include "pop_cmaes.hpp"
+#include <ensmallen_bits/function.hpp>
+
+namespace ens {
+
+template<typename SelectionPolicyType, typename TransformationPolicyType, bool UseBIPOPFlag>
+POP_CMAES<SelectionPolicyType,
+          TransformationPolicyType,
+          UseBIPOPFlag>::POP_CMAES(
+    const size_t lambda,
+    const TransformationPolicyType& transformationPolicy,
+    const size_t batchSize,
+    const size_t maxIterations,
+    const double tolerance,
+    const SelectionPolicyType& selectionPolicy,
+    double stepSize,
+    const size_t maxRestarts,
+    const double populationFactor,
+    const size_t maxFunctionEvaluations) :
+    CMAES<SelectionPolicyType, TransformationPolicyType>(
+        lambda, transformationPolicy, batchSize, maxIterations,
+        tolerance, selectionPolicy, stepSize),
+    populationFactor(populationFactor),
+    maxRestarts(maxRestarts),
+    maxFunctionEvaluations(maxFunctionEvaluations)
+{ /* Nothing to do. */ }
+
+template<typename SelectionPolicyType,
+         typename TransformationPolicyType,
+         bool UseBIPOPFlag>
+template<typename SeparableFunctionType, typename MatType, typename... CallbackTypes>
+typename MatType::elem_type POP_CMAES<SelectionPolicyType, 
+    TransformationPolicyType, UseBIPOPFlag>::Optimize(
+    SeparableFunctionType& function,
+    MatType& iterateIn,
+    CallbackTypes&&... callbacks)
+{
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+
+  StoreBestCoordinates<MatType> sbc;
+  StoreBestCoordinates<MatType> overallSBC;
+  size_t totalFunctionEvaluations = 0;
+  size_t largePopulationBudget = 0;
+  size_t smallPopulationBudget = 0;
+
+  // First single run with default population size
+  MatType iterate = iterateIn;
+  ElemType overallObjective = CMAES<SelectionPolicyType, 
+      TransformationPolicyType>::Optimize(function, iterate, sbc, 
+                                          callbacks...);
+
+  overallSBC = sbc;
+  ElemType objective;
+  size_t evaluations;
+
+  size_t defaultLambda = this->PopulationSize();
+  size_t currentLargeLambda = defaultLambda;
+
+  double stepSizeDefault = this->StepSize();
+
+  // Print out the default population size
+  Info << "Default population size: " << defaultLambda << "." << std::endl;
+
+  size_t restart = 0;
+
+  while (restart < maxRestarts)
+  {
+    if (!UseBIPOPFlag || largePopulationBudget <= smallPopulationBudget || 
+        restart == 0 || restart == maxRestarts - 1)
+    {
+      // Large population regime (IPOP or BIPOP)
+      currentLargeLambda *= populationFactor;
+      this->PopulationSize() = currentLargeLambda;
+      this->StepSize() = stepSizeDefault;
+
+      Info << "POP-CMA-ES: restart " << restart << ", large population size" <<
+          " (lambda): " << this->PopulationSize() << "." << std::endl;
+      
+      iterate = iterateIn;
+
+      // Optimize using the CMAES object.
+      objective = CMAES<SelectionPolicyType, 
+          TransformationPolicyType>::Optimize(function, iterate, sbc, 
+          callbacks...);
+
+      evaluations = this->FunctionEvaluations();
+      largePopulationBudget += evaluations;
+    }
+    else if (UseBIPOPFlag)
+    {
+      // Small population regime (BIPOP only)
+      double u = arma::randu<double>();
+      size_t smallLambda = static_cast<size_t>(defaultLambda * std::pow(0.5 * 
+          currentLargeLambda / defaultLambda, u * u));
+      double stepSizeSmall = 2 * std::pow(10, -2 * arma::randu<double>());
+      
+      this->PopulationSize() = smallLambda;
+      this->StepSize() = stepSizeSmall;
+
+      Info << "BIPOP-CMA-ES: restart " << restart << ", small population" <<
+          " size (lambda): " << this->PopulationSize() << "." << std::endl;
+
+      iterate = iterateIn;
+      
+      // Optimize using the CMAES object.
+      objective = CMAES<SelectionPolicyType, 
+          TransformationPolicyType>::Optimize(function, iterate, sbc, 
+                                              callbacks...);
+
+      evaluations = this->FunctionEvaluations();
+      smallPopulationBudget += evaluations;
+    }
+
+    if (objective < overallObjective)
+    {
+      overallObjective = objective;
+      overallSBC = sbc;
+      Info << "POP-CMA-ES: New best objective: " << overallObjective
+          << "." << std::endl;
+    }
+
+    totalFunctionEvaluations += evaluations;
+    // Check if the total number of evaluations has exceeded the limit
+    if (totalFunctionEvaluations >= maxFunctionEvaluations) {
+      Warn << "POP-CMA-ES: Maximum function overall evaluations reached. "
+           << "terminating optimization." << std::endl;
+
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
+      iterateIn = std::move(overallSBC.BestCoordinates());
+      return overallSBC.BestObjective();
+    }
+
+    ++restart;
+  }
+
+  Callback::EndOptimization(*this, function, iterate, callbacks...);
+  iterateIn = std::move(overallSBC.BestCoordinates());
+  return overallSBC.BestObjective();
+}
+
+} // namespace ens
+
+#endif
@@ -41,6 +41,7 @@ class RandomSelection
   * @tparam SeparableFunctionType Type of the function to be evaluated.
   * @param function Function to optimize.
   * @param batchSize Batch size to use for each step.
+   * @param terminate Whether optimization should be terminated after this call.
   * @param iterate starting point.
   */
  template<typename SeparableFunctionType,
@@ -49,6 +50,7 @@ class RandomSelection
  double Select(SeparableFunctionType& function,
                const size_t batchSize,
                const MatType& iterate,
+                bool& terminate,
                CallbackTypes&... callbacks)
  {
    // Find the number of functions to use.
@@ -64,7 +66,8 @@ class RandomSelection

      objective += function.Evaluate(iterate, selection, effectiveBatchSize);

-      Callback::Evaluate(*this, f, iterate, objective, callbacks...);
+      terminate |= Callback::Evaluate(*this, f, iterate, objective,
+          callbacks...);
    }

    return objective;
@@ -0,0 +1,162 @@
+/**
+ * @file boundary_box_constraint.hpp
+ * @author Suvarsha Chennareddy
+ *
+ * Boundary Box Transformation.
+ *
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CMAES_BOUNDARY_BOX_TRANSFORMATION_HPP
+#define ENSMALLEN_CMAES_BOUNDARY_BOX_TRANSFORMATION_HPP
+
+namespace ens {
+
+/**
+ * More often than not, coordinates must be bounded by some constraints.
+ * In a particular case, the domain of a specific function is restricted 
+ * by boundaries.
+ * The implemented transformation transforms given coordinates into a region 
+ * bounded by the given lower and upper bounds (a box). First, the 
+ * coordinates are shifted into a feasible preimage bounded by lowerBound - al
+ * and upperBound + au where al and au and calculated internally. 
+ * These shifted coordinates are then transformed into coordinates bounded by
+ * lower_bound and upper_bound. It is an identity transformation in between
+ * the lower and upper bounds.
+ * 
+ * For more information, check the original implementation in C by N. Hansen:
+ * https://github.com/CMA-ES/c-cmaes/blob/master/src/boundary_transformation.c
+ *
+ * @tparam MatType The matrix type of the coordinates and bounds.
+ */
+template<typename MatType = arma::mat>
+class BoundaryBoxConstraint
+{
+public:
+
+  /**
+   * Construct the boundary box constraint policy.
+   */
+  BoundaryBoxConstraint()
+  { /* Nothing to do. */ }
+
+  /**
+   * Construct the boundary box constraint policy.
+   *
+   * @param lowerBound The lower bound of the coordinates.
+   * @param upperBound The upper bound of the coordinates.
+   */
+  BoundaryBoxConstraint(const MatType& lowerBound,
+                        const MatType& upperBound) :
+      lowerBound(lowerBound),
+      upperBound(upperBound)
+  {}
+  
+  /**
+   * Construct the boundary box constraint policy.
+   *
+   * @param lowerBound The lower bound (for every dimension) of the coordinates.
+   * @param upperBound The upper bound (for every dimension) of the coordinates.
+   */
+  BoundaryBoxConstraint(const typename MatType::elem_type lowerBound,
+                        const typename MatType::elem_type upperBound) :
+      lowerBound({ (typename MatType::elem_type) lowerBound }),
+      upperBound({ (typename MatType::elem_type) upperBound })
+  {}
+
+  /**
+   * Map the given coordinates to the range 
+   * [lowerBound, upperBound]
+   *
+   * @param x Given coordinates.
+   * @return Transformed coordinates.
+   */
+  MatType Transform(const MatType& x)
+  {
+    typedef typename MatType::elem_type ElemType;
+    double diff, al, au, xlow, xup, r;
+    size_t Bi, Bj;
+    MatType y = x;
+    for (size_t i = 0; i < x.n_rows; i++)
+    {
+      Bi = (i < lowerBound.n_rows) ? i : (lowerBound.n_rows - 1);
+      for (size_t j = 0; j < x.n_cols; j++)
+      {
+        Bj = (j < lowerBound.n_cols) ? j : (lowerBound.n_cols - 1);
+
+        diff = (upperBound(Bi, Bj) - lowerBound(Bi, Bj)) / 2.0;
+        al = std::min(diff, (1 + std::abs(lowerBound(Bi, Bj))) / 20.0);
+        au = std::min(diff, (1 + std::abs(upperBound(Bi, Bj))) / 20.0);
+        xlow = lowerBound(Bi, Bj) - 2 * al - diff;
+        xup = upperBound(Bi, Bj) + 2 * au + diff;
+        r = 2 * (2 * diff + al + au);
+
+        // Shift y into feasible pre-image.
+        if (y(i, j) < xlow)
+        {
+          y(i,j) += (ElemType)(r * (1 + (int)((xlow - y(i, j)) / r)));
+        }
+        if (y(i, j) > xup)
+        {
+          y(i, j) -= (ElemType)(r * (1 + (int)((y(i, j) - xup) / r)));
+        }
+        if (y(i, j) < lowerBound(Bi, Bj) - al)
+        {
+          y(i, j) += (ElemType)(2 * (lowerBound(Bi, Bj) - al - y(i, j)));
+        }
+        if (y(i, j) > upperBound(Bi, Bj) + au)
+        {
+          y(i, j) -= (ElemType)(2 * (y(i, j) - upperBound(Bi, Bj) - au));
+        }
+
+        // Boundary transformation.
+        if (y(i, j) < lowerBound(Bi, Bj) + al)
+        {
+          y(i, j) = (ElemType)(lowerBound(Bi, Bj) +
+            (y(i, j) - (lowerBound(Bi, Bj) - al)) *
+            (y(i, j) - (lowerBound(Bi, Bj) - al)) / 4.0 / al);
+        }
+        else if (y(i,j) > upperBound(Bi,Bj) - au)
+        {
+          y(i, j) = (ElemType)(upperBound(Bi, Bj) -
+            (y(i, j) - (upperBound(Bi, Bj) + au)) *
+            (y(i, j) - (upperBound(Bi, Bj) + au)) / 4.0 / au);
+        }
+      }
+    }
+
+    return y;
+  }
+
+  /**
+   * Return a suitable initial step size.
+   *
+   * @return initial step size.
+   */
+  typename MatType::elem_type InitialStepSize()
+  { return 0.3 * (upperBound - lowerBound).min(); }
+
+  //! Get the lower bound of decision variables.
+  MatType LowerBound() const { return lowerBound; }
+  //! Modify the lower bound of decision variables.
+  MatType& LowerBound() { return lowerBound; }
+
+  //! Get the upper bound of decision variables.
+  MatType UpperBound() const { return upperBound; }
+  //! Modify the upper bound of decision variables.
+  MatType& UpperBound() { return upperBound; }
+
+private:
+  //! Lower bound of decision variables.
+  MatType lowerBound;
+
+  //! Upper bound of decision variables.
+  MatType upperBound;
+};
+
+} // namespace ens
+
+#endif
@@ -0,0 +1,47 @@
+/**
+ * @file empty_transformation.hpp
+ * @author Suvarsha Chennareddy
+ *
+ * Empty Transformation, can also be called an Indentity Transformation.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CMAES_EMPTY_TRANSFORMATION_HPP
+#define ENSMALLEN_CMAES_EMPTY_TRANSFORMATION_HPP
+
+namespace ens {
+
+/**
+ * This is an empty transformation. As the name indicates, it does
+ * not do anything. It is essentially an identity 
+ * transformation and is meant to be used when there are no
+ * sorts of constraints on the coordinates.
+ *
+ * @tparam MatType The matrix type of the coordinates.
+ */
+template<typename MatType = arma::mat>
+class EmptyTransformation
+{
+ public:
+  /**
+   * Transforms coordinates to themselves (effectively no transformation).
+   *
+   * @param x Input coordinates.
+   * @return Transformed coordinates (the coordinates themselves).
+   */
+  MatType Transform(const MatType& x) { return x; }
+
+  /**
+   * Return a suitable initial step size.
+   *
+   * @return initial step size.
+   */
+  typename MatType::elem_type InitialStepSize() { return 1; }
+};
+
+} // namespace ens
+
+#endif
@@ -111,26 +111,26 @@ typename MatType::elem_type CNE::Optimize(ArbitraryFunctionType& function,

  // Find the fitness before optimization using given iterate parameters.
  ElemType lastBestFitness = function.Evaluate(iterate);
-  Callback::Evaluate(*this, function, iterate, lastBestFitness, callbacks...);
+  terminate |= Callback::Evaluate(*this, function, iterate, lastBestFitness,
+      callbacks...);

  // Iterate until maximum number of generations is obtained.
-  terminate |= Callback::BeginOptimization(*this, function, iterate,
-      callbacks...);
+  Callback::BeginOptimization(*this, function, iterate, callbacks...);
  for (size_t gen = 1; gen <= maxGenerations && !terminate; gen++)
  {
    // Calculating fitness values of all candidates.
    for (size_t i = 0; i < populationSize; i++)
    {
-       // Select a candidate and insert the parameters in the function.
-       iterate = population[i];
-       terminate |= Callback::StepTaken(*this, function, iterate,
-          callbacks...);
+        // Select a candidate and insert the parameters in the function.
+        iterate = population[i];
+        terminate |= Callback::StepTaken(*this, function, iterate,
+            callbacks...);

-       // Find fitness of candidate.
-       fitnessValues[i] = function.Evaluate(iterate);
+        // Find fitness of candidate.
+        fitnessValues[i] = function.Evaluate(iterate);

-       Callback::Evaluate(*this, function, iterate, fitnessValues[i],
-          callbacks...);
+        terminate |= Callback::Evaluate(*this, function, iterate,
+            fitnessValues[i], callbacks...);
    }

    Info << "Generation number: " << gen << " best fitness = "
@@ -154,8 +154,10 @@ typename MatType::elem_type CNE::Optimize(ArbitraryFunctionType& function,
  // Set the best candidate into the network parameters.
  iterateIn = population[index(0)];

+  // The output of the callback doesn't matter because the optimization is
+  // finished.
  const ElemType objective = function.Evaluate(iterate);
-  Callback::Evaluate(*this, function, iterate, objective, callbacks...);
+  (void) Callback::Evaluate(*this, function, iterate, objective, callbacks...);

  Callback::EndOptimization(*this, function, iterate, callbacks...);
  return objective;
@@ -56,18 +56,8 @@
 #endif


-// Define ens_deprecated for deprecated functionality.
-// This is adapted from Armadillo's implementation.
-#if defined(_MSC_VER)
-  #define ens_deprecated __declspec(deprecated)
-#elif defined(__GNUG__) && (!defined(__clang__))
-  #define ens_deprecated __attribute__((__deprecated__))
-#elif defined(__clang__)
-  #if __has_attribute(__deprecated__)
-    #define ens_deprecated __attribute__((__deprecated__))
-  #else
-    #define ens_deprecated
-  #endif
-#else
-  #define ens_deprecated
+// undefine conflicting macros
+#if defined(As)
+  #pragma message ("WARNING: undefined conflicting 'As' macro")
+  #undef As
 #endif
@@ -77,8 +77,8 @@ typename MatType::elem_type DE::Optimize(FunctionType& function,
    population[i] += iterate;
    fitnessValues[i] = function.Evaluate(population[i]);

-    Callback::Evaluate(*this, function, population[i], fitnessValues[i],
-        callbacks...);
+    terminate |= Callback::Evaluate(*this, function, population[i],
+        fitnessValues[i], callbacks...);

    if (fitnessValues[i] < lastBestFitness)
    {
@@ -88,8 +88,7 @@ typename MatType::elem_type DE::Optimize(FunctionType& function,
  }

  // Iterate until maximum number of generations are completed.
-  terminate |= Callback::BeginOptimization(*this, function, iterate,
-      callbacks...);
+  Callback::BeginOptimization(*this, function, iterate, callbacks...);
  for (size_t gen = 0; gen < maxGenerations && !terminate; gen++)
  {
    // Generate new population based on /best/1/bin strategy.
@@ -126,10 +125,15 @@ typename MatType::elem_type DE::Optimize(FunctionType& function,
      }

      ElemType iterateValue = function.Evaluate(iterate);
-      Callback::Evaluate(*this, function, iterate, iterateValue, callbacks...);
+      terminate |= Callback::Evaluate(*this, function, iterate, iterateValue,
+          callbacks...);

      const ElemType mutantValue = function.Evaluate(mutant);
-      Callback::Evaluate(*this, function, mutant, mutantValue, callbacks...);
+      terminate |= Callback::Evaluate(*this, function, mutant, mutantValue,
+          callbacks...);
+
+      if (terminate)
+        break;

      // Replace the current member if mutant is better.
      if (mutantValue < iterateValue)
@@ -0,0 +1,207 @@
+/**
+ * @file demon_adam.hpp
+ * @author Marcus Edel
+ *
+ * Definition of DemonAdam.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DEMON_ADAM_DEMON_ADAM_HPP
+#define ENSMALLEN_DEMON_ADAM_DEMON_ADAM_HPP
+
+#include "../sgd/sgd.hpp"
+#include "../adam/adam_update.hpp"
+#include "../adam/adamax_update.hpp"
+#include "../adam/amsgrad_update.hpp"
+#include "../adam/nadam_update.hpp"
+#include "../adam/nadamax_update.hpp"
+#include "../adam/optimisticadam_update.hpp"
+#include "demon_adam_update.hpp"
+
+namespace ens {
+
+/**
+ * DemonAdam automatically decays momentum, motivated by decaying the total
+ * contribution of a gradient to all future updates.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{
+ *   title   = {Decaying momentum helps neural network training},
+ *   author  = {John Chen and Cameron Wolfe and Zhao Li
+ *              and Anastasios Kyrillidis},
+ *   url     = {https://arxiv.org/abs/1910.04952}
+ *   year    = {2019}
+ * }
+ *
+ * DemonAdam can optimize differentiable separable functions. For more details,
+ * see the documentation on function types include with this distribution or on
+ * the ensmallen website.
+ *
+ * @tparam UpdateRule Adam optimizer update rule to be used.
+ */
+template<typename UpdateRule = AdamUpdate>
+class DemonAdamType
+{
+ public:
+  /**
+   * Construct the DemonAdam optimizer with the given function and parameters.
+   * The defaults here are not necessarily good for the given problem, so it is
+   * suggested that the values used be tailored to the task at hand.  The
+   * maximum number of iterations refers to the maximum number of points that
+   * are processed (i.e., one iteration equals one point; one iteration does not
+   * equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in a single step.
+   * @param momentum The initial momentum coefficient.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *     limit).
+   * @param beta1 Exponential decay rate for the first moment estimates.
+   * @param beta2 Exponential decay rate for the weighted infinity norm
+   *     estimates.
+   * @param eps Value used to initialise the mean squared gradient parameter.
+      * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *     function is visited in linear order.
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *     call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *     final objective obtained on the last pass over the data).
+   */
+  DemonAdamType(const double stepSize = 0.001,
+                const size_t batchSize = 32,
+                const double momentum = 0.9,
+                const double beta1 = 0.9,
+                const double beta2 = 0.999,
+                const double eps = 1e-8,
+                const size_t maxIterations = 100000,
+                const double tolerance = 1e-5,
+                const bool shuffle = true,
+                const bool resetPolicy = true,
+                const bool exactObjective = false) :
+      optimizer(stepSize,
+                batchSize,
+                maxIterations,
+                tolerance,
+                shuffle,
+                DemonAdamUpdate<UpdateRule>(maxIterations * batchSize,
+                    momentum, UpdateRule(eps, beta1, beta2)),
+                NoDecay(),
+                resetPolicy,
+                exactObjective)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Optimize the given function using DemonAdam. The given starting point will
+   * be modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return optimizer.template Optimize<
+        SeparableFunctionType, MatType, GradType, CallbackTypes...>(
+        function, iterate, std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the moment coefficient.
+  double Momentum() const { return optimizer.UpdatePolicy().Momentum(); }
+  //! Modify the moment coefficient.
+  double& Momentum() { return optimizer.UpdatePolicy().Momentum(); }
+
+  //! Get the momentum iteration number.
+  size_t MomentumIterations() const
+  { return optimizer.UpdatePolicy().MomentumIterations(); }
+  //! Modify the momentum iteration number.
+  size_t& MomentumIterations()
+  { return optimizer.UpdatePolicy().MomentumIterations(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
+  //! Get whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+ private:
+  //! The Stochastic Gradient Descent object with DemonAdam policy.
+  SGD<DemonAdamUpdate<UpdateRule>> optimizer;
+};
+
+using DemonAdam = DemonAdamType<AdamUpdate>;
+
+using DemonAdaMax = DemonAdamType<AdaMaxUpdate>;
+
+using DemonAMSGrad = DemonAdamType<AMSGradUpdate>;
+
+using DemonNadam = DemonAdamType<NadamUpdate>;
+
+using DemonNadaMax = DemonAdamType<NadaMaxUpdate>;
+
+using DemonOptimisticAdam = DemonAdamType<OptimisticAdamUpdate>;
+
+} // namespace ens
+
+#endif
@@ -0,0 +1,169 @@
+/**
+ * @file demon_sgd_update.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of DemonAdam.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DEMON_ADAM_DEMON_ADAM_UPDATE_HPP
+#define ENSMALLEN_DEMON_ADAM_DEMON_ADAM_UPDATE_HPP
+
+#include <assert.h>
+
+namespace ens {
+
+/**
+ * DemonAdam automatically decays momentum, motivated by decaying the total
+ * contribution of a gradient to all future updates.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{
+ *   title   = {Decaying momentum helps neural network training},
+ *   author  = {John Chen and Cameron Wolfe and Zhao Li
+ *              and Anastasios Kyrillidis},
+ *   url     = {https://arxiv.org/abs/1910.04952}
+ *   year    = {2019}
+ * }
+ * @endcode
+ *
+ * @tparam UpdateRule DemonAdam optimizer update rule to be used.
+ */
+template<typename UpdateRule>
+class DemonAdamUpdate
+{
+ public:
+  /**
+   * Construct the DemonAdam update policy with the given parameters.
+   *
+   * @param momentumIterations The number of iterations before the momentum
+   *     will decay to zero.
+   * @param momentum The initial momentum coefficient.
+   * @param adamUpdate Instantiated Adam update policy used to adjust the given
+   *     parameters.
+   */
+  DemonAdamUpdate(const size_t momentumIterations = 100,
+                  const double momentum = 0.9,
+                  const UpdateRule& adamUpdate = UpdateRule()) :
+      T(momentumIterations),
+      betaInit(momentum),
+      t(0),
+      adamUpdateInst(adamUpdate)
+  {
+    // Make sure the momentum iterations parameter is non-zero.
+    assert(momentumIterations != 0 && "The number of iterations before the "
+        "momentum will decay is zero, make sure the max iterations and "
+        "batch size parameter is set correctly. "
+        "Default: momentumIterations = maxIterations / batchSize.");
+  }
+
+  //! Get the momentum coefficient.
+  double Momentum() const { return betaInit; }
+  //! Modify the momentum coefficient.
+  double& Momentum() { return betaInit; }
+
+  //! Get the current iteration number.
+  size_t Iteration() const { return t; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return t; }
+
+  //! Get the momentum ion number.
+  size_t MomentumIterations() const { return T; }
+  //! Modify the momentum iteration number.
+  size_t& MomentumIterations() { return T; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    // Convenient typedef.
+    typedef typename UpdateRule::template Policy<MatType, GradType>
+        InstUpdateRuleType;
+
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent Instantiated PadamUpdate parent object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(DemonAdamUpdate& parent,
+           const size_t rows,
+           const size_t cols) :
+      parent(parent),
+      adamUpdate(new InstUpdateRuleType(parent.adamUpdateInst, rows, cols))
+    { /* Nothing to do here */ }
+
+    /**
+     * Clean any memory associated with the Polciy object.
+     */
+    ~Policy()
+    {
+      delete adamUpdate;
+    }
+
+    /**
+     * Update step for DamonAdam.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      double decayRate = 1;
+      if (parent.t > 0)
+        decayRate = 1.0 - (double) parent.t / (double) parent.T;
+
+      const double betaDecay = parent.betaInit * decayRate;
+      const double beta = betaDecay / ((1.0 - parent.betaInit) + betaDecay);
+
+      // Perform the update.
+      iterate *= beta;
+
+      // Apply the adam update.
+      adamUpdate->Update(iterate, stepSize, gradient);
+
+      // Increment the iteration counter variable.
+      ++parent.t;
+    }
+
+   private:
+    //! Instantiated parent object.
+    DemonAdamUpdate<UpdateRule>& parent;
+
+    //! The update policy.
+    InstUpdateRuleType* adamUpdate;
+  };
+
+ private:
+  //! The number of momentum iterations.
+  size_t T;
+
+  //! Initial momentum coefficient.
+  double betaInit;
+
+  //! The number of iterations.
+  size_t t;
+
+  //! The adam update policy.
+  UpdateRule adamUpdateInst;
+};
+
+} // namespace ens
+
+#endif
@@ -0,0 +1,178 @@
+/**
+ * @file demon_sgd.hpp
+ * @author Marcus Edel
+ *
+ * Definition of DemonSGD.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DEMON_SGD_DEMON_SGD_HPP
+#define ENSMALLEN_DEMON_SGD_DEMON_SGD_HPP
+
+#include "../sgd/sgd.hpp"
+#include "demon_sgd_update.hpp"
+
+namespace ens {
+
+/**
+ * DemonSGD automatically decays momentum, motivated by decaying the total
+ * contribution of a gradient to all future updates.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{
+ *   title   = {Decaying momentum helps neural network training},
+ *   author  = {John Chen and Cameron Wolfe and Zhao Li
+ *              and Anastasios Kyrillidis},
+ *   url     = {https://arxiv.org/abs/1910.04952}
+ *   year    = {2019}
+ * }
+ *
+ * DemonSGD can optimize differentiable separable functions. For more details,
+ * see the documentation on function types include with this distribution or on
+ * the ensmallen website.
+ */
+class DemonSGD
+{
+ public:
+  /**
+   * Construct the DemonSGD optimizer with the given function and parameters.
+   * The defaults here are not necessarily good for the given problem, so it is
+   * suggested that the values used be tailored to the task at hand.  The
+   * maximum number of iterations refers to the maximum number of points that
+   * are processed (i.e., one iteration equals one point; one iteration does not
+   * equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in a single step.
+   * @param momentum The initial momentum coefficient.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *     limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *     function is visited in linear order.
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *     call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *     final objective obtained on the last pass over the data).
+   */
+  DemonSGD(const double stepSize = 0.001,
+           const size_t batchSize = 32,
+           const double momentum = 0.9,
+           const size_t maxIterations = 100000,
+           const double tolerance = 1e-5,
+           const bool shuffle = true,
+           const bool resetPolicy = true,
+           const bool exactObjective = false) :
+      optimizer(stepSize,
+                batchSize,
+                maxIterations,
+                tolerance,
+                shuffle,
+                DemonSGDUpdate(maxIterations * batchSize, momentum),
+                NoDecay(),
+                resetPolicy,
+                exactObjective)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Optimize the given function using DemonSGD. The given starting point will
+   * be modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return optimizer.template Optimize<
+        SeparableFunctionType, MatType, GradType, CallbackTypes...>(
+        function, iterate, std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the moment coefficient.
+  double Momentum() const { return optimizer.UpdatePolicy().Momentum(); }
+  //! Modify the moment coefficient.
+  double& Momentum() { return optimizer.UpdatePolicy().Momentum(); }
+
+  //! Get the momentum iteration number.
+  size_t MomentumIterations() const
+  { return optimizer.UpdatePolicy().MomentumIterations(); }
+  //! Modify the momentum iteration number.
+  size_t& MomentumIterations()
+  { return optimizer.UpdatePolicy().MomentumIterations(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
+  //! Get whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+ private:
+  //! The Stochastic Gradient Descent object with DemonSGD policy.
+  SGD<DemonSGDUpdate> optimizer;
+};
+
+} // namespace ens
+
+#endif
@@ -0,0 +1,139 @@
+/**
+ * @file demon_sgd_update.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of DemonSGD.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DEMON_SGD_DEMON_SGD_UPDATE_HPP
+#define ENSMALLEN_DEMON_SGD_DEMON_SGD_UPDATE_HPP
+
+namespace ens {
+
+/**
+ * DemonSGD automatically decays momentum, motivated by decaying the total
+ * contribution of a gradient to all future updates.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{
+ *   title   = {Decaying momentum helps neural network training},
+ *   author  = {John Chen and Cameron Wolfe and Zhao Li
+ *              and Anastasios Kyrillidis},
+ *   url     = {https://arxiv.org/abs/1910.04952}
+ *   year    = {2019}
+ * }
+ * @endcode
+ */
+class DemonSGDUpdate
+{
+ public:
+  /**
+   * Construct the DemonSGD update policy with the given parameters.
+   *
+   * @param momentumIterations The number of iterations before the momentum
+   *     will decay to zero.
+   * @param momentum The initial momentum coefficient.
+   */
+  DemonSGDUpdate(const size_t momentumIterations = 100,
+                 const double momentum = 0.9) :
+      T(momentumIterations),
+      betaInit(momentum),
+      t(0)
+  {
+    // Make sure the momentum iterations parameter is non-zero.
+    assert(momentumIterations != 0 && "The number of iterations before the "
+        "momentum will decay is zero, make sure the max iterations and "
+        "batch size parameter is set correctly. "
+        "Default: momentumIterations = maxIterations * batchSize.");
+  }
+
+  //! Get the momentum coefficient.
+  double Momentum() const { return betaInit; }
+  //! Modify the momentum coefficient.
+  double& Momentum() { return betaInit; }
+
+  //! Get the current iteration number.
+  size_t Iteration() const { return t; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return t; }
+
+  //! Get the momentum iteration number.
+  size_t MomentumIterations() const { return T; }
+  //! Modify the momentum iteration number.
+  size_t& MomentumIterations() { return T; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent Instantiated PadamUpdate parent object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(DemonSGDUpdate& parent,
+           const size_t /* rows */,
+           const size_t /* cols */) :
+      parent(parent)
+    { /* Nothing to do here */ }
+
+    /**
+     * Update step for DemonSGD.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      double decayRate = 1;
+      if (parent.t > 0)
+        decayRate = 1.0 - (double) parent.t / (double) parent.T;
+
+      const double betaDecay = parent.betaInit * decayRate;
+      const double beta = betaDecay / ((1.0 - parent.betaInit) + betaDecay);
+
+      // Perform the update.
+      iterate *= beta;
+      iterate -= stepSize * gradient;
+
+      // Increment the iteration counter variable.
+      ++parent.t;
+    }
+
+   private:
+    //! Instantiated parent object.
+    DemonSGDUpdate& parent;
+  };
+
+ private:
+  //! The number of momentum iterations.
+  size_t T;
+
+  //! Initial momentum coefficient.
+  double betaInit;
+
+  //! The number of iterations.
+  size_t t;
+};
+
+} // namespace ens
+
+#endif
@@ -15,17 +15,17 @@
 #define ENS_VERSION_MAJOR 2
 // The minor version is two digits so regular numerical comparisons of versions
 // work right.  The first minor version of a release is always 10.
-#define ENS_VERSION_MINOR 16
-#define ENS_VERSION_PATCH 2
+#define ENS_VERSION_MINOR 22
+#define ENS_VERSION_PATCH 0
 // If this is a release candidate, it will be reflected in the version name
 // (i.e. the version name will be "RC1", "RC2", etc.).  Otherwise the version
 // name will typically be a seemingly arbitrary set of words that does not
 // contain the capitalized string "RC".
-#define ENS_VERSION_NAME "Severely Dented Can Of Polyurethane"
+#define ENS_VERSION_NAME "E-Bike Excitement"
 // Incorporate the date the version was released.
-#define ENS_VERSION_YEAR "2021"
-#define ENS_VERSION_MONTH "03"
-#define ENS_VERSION_DAY "24"
+#define ENS_VERSION_YEAR "2024"
+#define ENS_VERSION_MONTH "11"
+#define ENS_VERSION_DAY "29"

 namespace ens {

@@ -98,7 +98,7 @@ Eve::Optimize(SeparableFunctionType& function,
  v.zeros();

  // Now iterate!
-  terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  Callback::BeginOptimization(*this, f, iterate, callbacks...);
  BaseGradType gradient(iterate.n_rows, iterate.n_cols);
  const size_t actualMaxIterations = (maxIterations == 0) ?
      std::numeric_limits<size_t>::max() : maxIterations;
@@ -123,6 +123,8 @@ Eve::Optimize(SeparableFunctionType& function,

    terminate |= Callback::EvaluateWithGradient(*this, f, iterate,
        objective, gradient, callbacks...);
+    if (terminate)
+      break;

    m *= beta1;
    m += (1 - beta1) * gradient;
@@ -172,9 +174,7 @@ Eve::Optimize(SeparableFunctionType& function,
        return overallObjective;
      }

-      if (std::abs(lastOverallObjective - overallObjective) < tolerance ||
-          Callback::BeginEpoch(*this, f, iterate, epoch, overallObjective,
-              callbacks...))
+      if (std::abs(lastOverallObjective - overallObjective) < tolerance)
      {
        Info << "Eve: minimized within tolerance " << tolerance << "; "
            << "terminating optimization." << std::endl;
@@ -183,6 +183,9 @@ Eve::Optimize(SeparableFunctionType& function,
        return overallObjective;
      }

+      terminate |= Callback::BeginEpoch(*this, f, iterate, epoch,
+          overallObjective, callbacks...);
+
      // Reset the counter variables.
      lastOverallObjective = overallObjective;
      overallObjective = 0;
@@ -206,7 +209,9 @@ Eve::Optimize(SeparableFunctionType& function,
      const ElemType objective = f.Evaluate(iterate, i, effectiveBatchSize);
      overallObjective += objective;

-      Callback::Evaluate(*this, f, iterate, objective, callbacks...);
+      // The optimization is finished, so we don't need to care about the result
+      // of the callback.
+      (void) Callback::Evaluate(*this, f, iterate, objective, callbacks...);
    }
  }

@@ -50,8 +50,7 @@ class FTMLUpdate
             const double beta2 = 0.999) :
      epsilon(epsilon),
      beta1(beta1),
-      beta2(beta2),
-      iteration(0)
+      beta2(beta2)
  { /* Do nothing. */ }

  //! Get the value used to initialise the squared gradient parameter.
@@ -69,11 +68,6 @@ class FTMLUpdate
  //! Modify the second moment coefficient.
  double& Beta2() { return beta2; }

-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
  /**
   * The UpdatePolicyType policy classes must contain an internal 'Policy'
   * template class with two template arguments: MatType and GradType.  This is
@@ -112,16 +106,14 @@ class FTMLUpdate
                const GradType& gradient)
    {
      // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;

      // And update the iterate.
      v *= parent.beta2;
      v += (1 - parent.beta2) * (gradient % gradient);

-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);

      MatType sigma = -parent.beta1 * d;
      d = biasCorrection1 / stepSize *
@@ -145,6 +137,9 @@ class FTMLUpdate

    // Parameter update term.
    MatType d;
+
+    // The number of iterations.
+    size_t iteration;
  };

 private:
@@ -156,9 +151,6 @@ class FTMLUpdate

  // The second moment coefficient.
  double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };

 } // namespace ens
@@ -77,11 +77,6 @@ struct MatTypeTraits<arma::SpSubview<eT>>
 };


-#if ((ARMA_VERSION_MAJOR >= 10) || \
-    ((ARMA_VERSION_MAJOR == 9) && (ARMA_VERSION_MINOR >= 869)))
-
-// Armadillo 9.869+ has SpSubview_col and SpSubview_row
-
 template<typename eT>
 struct MatTypeTraits<arma::SpSubview_col<eT>>
 {
@@ -98,9 +93,6 @@ struct MatTypeTraits<arma::SpSubview_row<eT>>
      "or a matrix alias instead!");
 };

-#endif
-
-
 template<typename eT>
 struct MatTypeTraits<arma::Cube<eT>>
 {
@@ -188,7 +188,7 @@ struct NAME                                                                    \
    template<typename>                                                         \
    static no& chk(...);                                                       \
                                                                               \
-    static const bool value = sizeof(chk<Class>(0)) == sizeof(yes);            \
+    static constexpr bool value = sizeof(chk<Class>(0)) == sizeof(yes);        \
  };                                                                           \
                                                                               \
  template<size_t N>                                                           \
@@ -201,10 +201,10 @@ struct NAME                                                                    \
            N < MAXN,                                                          \
            WithGreaterOrEqualNumberOfAdditionalArgs<N + 1>,                   \
            std::false_type>::type>::type;                                     \
-    static const bool value = type::value;                                     \
+    static constexpr bool value = type::value;                                 \
  };                                                                           \
                                                                               \
-  static const bool value =                                                    \
+  static constexpr bool value =                                                \
      WithGreaterOrEqualNumberOfAdditionalArgs<MinN>::value;                   \
 };

@@ -235,7 +235,7 @@ struct NAME                                                                  \
  template <typename Q = T>                                                  \
  static char f(char) { return 0; }                                        \
                                                                             \
-  static const bool value = sizeof(f<T>(0)) != sizeof(char);                 \
+  static constexpr bool value = sizeof(f<T>(0)) != sizeof(char);             \
 };
 /*
 * A macro that can be used for passing arguments containing commas to other
@@ -387,6 +387,25 @@ inline void CheckArbitraryFunctionTypeAPI()
 #endif
 }

+template<typename FunctionType, typename... RemainingTypes>
+typename std::enable_if<(sizeof...(RemainingTypes) > 1), void>::type
+CheckArbitraryFunctionTypeAPI()
+{
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  constexpr size_t size = sizeof...(RemainingTypes);
+  using TupleType = typename std::tuple<RemainingTypes...>;
+  using MatType = typename std::tuple_element<size - 1, TupleType>::type;
+
+  static_assert(CheckEvaluate<FunctionType, MatType, MatType>::value,
+      "One of the provided FunctionType does not have a correct definition of Evaluate(). "
+      "Please check that the corresponding FunctionType fully satisfies the requirements "
+      "of the ArbitraryFunctionType API; see the optimizer tutorial for "
+      "more details.");
+
+  CheckArbitraryFunctionTypeAPI<RemainingTypes...>();
+#endif
+}
+
 /**
 * Perform checks for the ResolvableFunctionType API.
 */
@@ -96,8 +96,7 @@ class Atoms
      // Find possible atom to be deleted.
      arma::vec gap = sqTerm -
          currentCoeffs % trans(gradient.t() * currentAtoms);
-      arma::uword ind;
-      gap.min(ind);
+      arma::uword ind = gap.index_min();

      // Try deleting the atom.
      arma::mat newAtoms(currentAtoms.n_rows, currentAtoms.n_cols - 1);
@@ -118,8 +118,8 @@ class ConstrLpBallSolver
      else
        s = arma::abs(v);

-      arma::uword k = 0;
-      s.max(k);  // k is the linear index of the largest element.
+      // k is the linear index of the largest element.
+      arma::uword k = s.index_max();
      s.zeros();
      // Take the sign of v(k).
      s(k) = -((0.0 < v(k)) - (v(k) < 0.0));
@@ -75,7 +75,7 @@ FrankWolfe<LinearConstrSolverType, UpdateRuleType>::Optimize(
  // Controls early termination of the optimization process.
  bool terminate = false;

-  terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  Callback::BeginOptimization(*this, f, iterate, callbacks...);
  for (size_t i = 1; i != maxIterations && !terminate; ++i)
  {
    currentObjective = f.EvaluateWithGradient(iterate, gradient);
@@ -45,7 +45,7 @@ inline void Proximal::ProjectToL1Ball(MatType& v, double tau)
  MatType simplexSum = arma::cumsum(simplexSol);

  double nu = 0;
-  size_t rho = 0;
+  size_t rho = simplexSol.n_rows - 1;
  for (size_t j = 1; j <= simplexSol.n_rows; j++)
  {
    rho = simplexSol.n_rows - j;
@@ -53,7 +53,7 @@ inline void Proximal::ProjectToL1Ball(MatType& v, double tau)
    if (nu > 0)
      break;
  }
-  double theta = (simplexSum(rho) - tau) / rho;
+  const double theta = (simplexSum(rho) - tau) / rho;

  // Threshold on absolute value of v with theta.
  for (arma::uword j = 0; j < simplexSol.n_rows; j++)
@@ -66,7 +66,7 @@ GradientDescent::Optimize(FunctionType& function,
  bool terminate = false;

  // Now iterate!
-  terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  Callback::BeginOptimization(*this, f, iterate, callbacks...);
  for (size_t i = 1; i != maxIterations && !terminate; ++i)
  {
    overallObjective = f.EvaluateWithGradient(iterate, gradient);
@@ -112,8 +112,7 @@ IQN::Optimize(SeparableFunctionType& functionIn,
  BaseGradType gradient(iterate.n_rows, iterate.n_cols);
  BaseMatType u = t[0];

-  terminate |= Callback::BeginOptimization(*this, function, iterate,
-      callbacks...);
+  Callback::BeginOptimization(*this, function, iterate, callbacks...);
  for (size_t i = 1; i != maxIterations && !terminate; ++i)
  {
    for (size_t j = 0, f = 0; f < numFunctions; j++)
@@ -175,7 +174,7 @@ IQN::Optimize(SeparableFunctionType& functionIn,
          effectiveBatchSize);
      overallObjective += objective;

-      Callback::Evaluate(*this, function, iterate, objective,
+      terminate |= Callback::Evaluate(*this, function, iterate, objective,
          callbacks...);
    }
    overallObjective /= numFunctions;
@@ -112,8 +112,7 @@ KatyushaType<Proximal>::Optimize(

  const size_t actualMaxIterations = (maxIterations == 0) ?
      std::numeric_limits<size_t>::max() : maxIterations;
-  terminate |= Callback::BeginOptimization(*this, function, iterate,
-      callbacks...);
+  Callback::BeginOptimization(*this, function, iterate, callbacks...);
  for (size_t i = 0; i < actualMaxIterations && !terminate; ++i)
  {
    // Calculate the objective function.
@@ -125,7 +124,8 @@ KatyushaType<Proximal>::Optimize(
          effectiveBatchSize);
      overallObjective += objective;

-      Callback::Evaluate(*this, function, iterate0, objective, callbacks...);
+      terminate |= Callback::Evaluate(*this, function, iterate0, objective,
+          callbacks...);
    }

    if (std::isnan(overallObjective) || std::isinf(overallObjective))
@@ -174,7 +174,7 @@ KatyushaType<Proximal>::Optimize(
    double cw = 1;
    w.zeros();

-    for (size_t f = 0, currentFunction = 0; f < innerIterations;
+    for (size_t f = 0, currentFunction = 0; (f < innerIterations) && !terminate;
        /* incrementing done manually */)
    {
      // Is this iteration the start of a sequence?
@@ -254,7 +254,10 @@ KatyushaType<Proximal>::Optimize(
          effectiveBatchSize);
      overallObjective += objective;

-      Callback::Evaluate(*this, function, iterate, objective, callbacks...);
+      // The optimization is finished, so we don't need to care about the
+      // callback result.
+      (void) Callback::Evaluate(*this, function, iterate, objective,
+          callbacks...);
    }
  }

@@ -79,18 +79,27 @@ double L_BFGS::ChooseScalingFactor(const size_t iterationNum,
 {
  typedef typename CubeType::elem_type CubeElemType;

-  double scalingFactor = 1.0;
+  constexpr const CubeElemType tol =
+      100 * std::numeric_limits<CubeElemType>::epsilon();
+
+  double scalingFactor;
  if (iterationNum > 0)
  {
    int previousPos = (iterationNum - 1) % numBasis;
    // Get s and y matrices once instead of multiple times.
    const arma::Mat<CubeElemType>& sMat = s.slice(previousPos);
    const arma::Mat<CubeElemType>& yMat = y.slice(previousPos);
-    scalingFactor = dot(sMat, yMat) / dot(yMat, yMat);
+
+    const CubeElemType tmp   = arma::dot(yMat, yMat);
+    const CubeElemType denom = (tmp >= tol) ? tmp : CubeElemType(1);
+
+    scalingFactor = arma::dot(sMat, yMat) / denom;
  }
  else
  {
-    scalingFactor = 1.0 / sqrt(dot(gradient, gradient));
+    const CubeElemType tmp = arma::norm(gradient, "fro");
+
+    scalingFactor = (tmp >= tol) ? (1.0 / tmp) : 1.0;
  }

  return scalingFactor;
@@ -129,11 +138,19 @@ void L_BFGS::SearchDirection(const MatType& gradient,
  for (size_t i = iterationNum; i != limit; i--)
  {
    int translatedPosition = (i + (numBasis - 1)) % numBasis;
-    rho[iterationNum - i] = 1.0 / arma::dot(y.slice(translatedPosition),
-                                            s.slice(translatedPosition));
+
+    const arma::Mat<CubeElemType>& sMat = s.slice(translatedPosition);
+    const arma::Mat<CubeElemType>& yMat = y.slice(translatedPosition);
+
+    const CubeElemType tmp = arma::dot(yMat, sMat);
+
+    rho[iterationNum - i] = (tmp != CubeElemType(0)) ? (1.0 / tmp) :
+        CubeElemType(1);
+
    alpha[iterationNum - i] = rho[iterationNum - i] *
-        arma::dot(s.slice(translatedPosition), searchDirection);
-    searchDirection -= alpha[iterationNum - i] * y.slice(translatedPosition);
+        arma::dot(sMat, searchDirection);
+
+    searchDirection -= alpha[iterationNum - i] * yMat;
  }

  searchDirection *= scalingFactor;
@@ -218,7 +235,8 @@ bool L_BFGS::LineSearch(FunctionType& function,
      arma::dot(gradient, searchDirection);

  // If it is not a descent direction, just report failure.
-  if (initialSearchDirectionDotGradient > 0.0)
+  if ( (initialSearchDirectionDotGradient > 0.0)
+    || (std::isfinite(initialSearchDirectionDotGradient) == false) )
  {
    Warn << "L-BFGS line search direction is not a descent direction "
        << "(terminating)!" << std::endl;
@@ -250,6 +268,12 @@ bool L_BFGS::LineSearch(FunctionType& function,
    newIterateTmp += stepSize * searchDirection;
    functionValue = function.EvaluateWithGradient(newIterateTmp, gradient);

+    if (std::isnan(functionValue))
+    {
+      Warn << "L-BFGS objective value is NaN (terminating)!" << std::endl;
+      return false;
+    }
+
    terminate |= Callback::EvaluateWithGradient(*this, function, newIterateTmp,
        functionValue, gradient, callbacks...);

@@ -378,10 +402,10 @@ L_BFGS::Optimize(FunctionType& function,
  terminate |= Callback::EvaluateWithGradient(*this, f, iterate,
        functionValue, gradient, callbacks...);

-  ElemType prevFunctionValue = functionValue;
+  ElemType prevFunctionValue;

  // The main optimization loop.
-  terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  Callback::BeginOptimization(*this, f, iterate, callbacks...);
  for (size_t itNum = 0; (optimizeUntilConvergence || (itNum != maxIterations))
      && !terminate; ++itNum)
  {
@@ -391,6 +415,8 @@ L_BFGS::Optimize(FunctionType& function,
    //
    // But don't do this on the first iteration to ensure we always take at
    // least one descent step.
+    // TODO: to speed this up, investigate use of arma::norm2est() in Armadillo
+    // 12.4
    if (arma::norm(gradient, 2) < minGradientNorm)
    {
      Info << "L-BFGS gradient norm too small (terminating successfully)."
@@ -416,6 +442,13 @@ L_BFGS::Optimize(FunctionType& function,
      break;
    }

+    if (std::isfinite(scalingFactor) == false)
+      {
+      Warn << "L-BFGS scaling factor is not finite.  Stopping optimization."
+           << std::endl;
+      break;
+      }
+
    // Build an approximation to the Hessian and choose the search
    // direction for the current iteration.
    SearchDirection(gradient, itNum, scalingFactor, s, y, searchDirection);
@@ -133,7 +133,7 @@ Lookahead<BaseOptimizerType, DecayPolicyType>::Optimize(
  }

  // Now iterate!
-  terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  Callback::BeginOptimization(*this, f, iterate, callbacks...);
  const size_t actualMaxIterations = (maxIterations == 0) ?
      std::numeric_limits<size_t>::max() : maxIterations;
  for (size_t i = 0; i < actualMaxIterations && !terminate; i++)
@@ -198,7 +198,9 @@ Lookahead<BaseOptimizerType, DecayPolicyType>::Optimize(
      const ElemType objective = f.Evaluate(iterate, i, effectiveBatchSize);
      overallObjective += objective;

-      Callback::Evaluate(*this, f, iterate, objective, callbacks...);
+      // The optimization is over, so we don't need to care about the result of
+      // the callback.
+      (void) Callback::Evaluate(*this, f, iterate, objective, callbacks...);
    }
  }

@@ -0,0 +1,81 @@
+/**
+ * @file pbi_decomposition.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * The Penalty Based Boundary Intersection (PBI) decomposition policy.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOEAD_PBI_HPP
+#define ENSMALLEN_MOEAD_PBI_HPP
+
+namespace ens {
+
+/**
+ * Penalty Based Boundary Intersection (PBI) method is a weight decomposition method,
+ * it tries to find the intersection between bottom-most boundary of the attainable
+ * objective set with the reference directions.
+ *
+ * The goal is to minimize the distance between objective vectors with the ideal point
+ * along the reference direction. To handle equality constraints, a penalty parameter
+ * theta is used.
+ *
+ * For more information, see the following:
+ * @code
+ * article{zhang2007moea,
+ *   title={MOEA/D: A multiobjective evolutionary algorithm based on decomposition},
+ *   author={Zhang, Qingfu and Li, Hui},
+ *   journal={IEEE Transactions on evolutionary computation},
+ *   pages={712--731},
+ *   year={2007}
+ * @endcode
+ */
+class PenaltyBoundaryIntersection
+{
+ public:
+  /**
+   * Constructor for Penalty Based Boundary Intersection decomposition
+   * policy.
+   *
+   * @param theta The penalty value.
+   */
+  PenaltyBoundaryIntersection(const double theta = 5) :
+      theta(theta)
+  {
+    /* Nothing to do. */
+  }
+
+  /**
+   * Decompose the weight vectors.
+   *
+   * @tparam VecType The type of the vector used in the decommposition.
+   * @param weight The weight vector corresponding to a subproblem.
+   * @param idealPoint The reference point in the objective space.
+   * @param candidateFitness The objective vector of the candidate.
+   */
+  template<typename VecType>
+  typename VecType::elem_type Apply(const VecType& weight,
+                                    const VecType& idealPoint,
+                                    const VecType& candidateFitness)
+  {
+    typedef typename VecType::elem_type ElemType;
+    //! A unit vector in the same direction as the provided weight vector.
+    const VecType referenceDirection = weight / arma::norm(weight);
+    //! Distance of F(x) from the idealPoint along the reference direction.
+    const ElemType d1 = arma::dot(candidateFitness - idealPoint, referenceDirection);
+    //! The perpendicular distance of F(x) from reference direction.
+    const ElemType d2 = arma::norm(candidateFitness - (idealPoint + d1 * referenceDirection));
+
+    return d1 + static_cast<ElemType>(theta) * d2;
+  }
+
+  private:
+    double theta;
+};
+
+} // namespace ens
+
+#endif
@@ -0,0 +1,66 @@
+/**
+ * @file tchebycheff_decomposition.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * The Tchebycheff Weight decomposition policy.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOEAD_TCHEBYCHEFF_HPP
+#define ENSMALLEN_MOEAD_TCHEBYCHEFF_HPP
+
+namespace ens {
+
+/**
+ * The Tchebycheff method works by taking the maximum of element-wise product
+ * between reference direction and the line connecting objective vector and
+ * ideal point.
+ *
+ * Under mild conditions, for each Pareto Optimal point there exists a reference
+ * direction such that the given point is also the optimal solution
+ * to this scalar objective.
+ *
+ * For more information, see the following:
+ * @code
+ * article{zhang2007moea,
+ *   title={MOEA/D: A multiobjective evolutionary algorithm based on decomposition},
+ *   author={Zhang, Qingfu and Li, Hui},
+ *   journal={IEEE Transactions on evolutionary computation},
+ *   pages={712--731},
+ *   year={2007}
+ * @endcode
+ */
+class Tchebycheff
+{
+ public:
+  /**
+   * Constructor for Tchebycheff decomposition policy.
+   */
+  Tchebycheff()
+  {
+    /* Nothing to do. */
+  }
+
+  /**
+   * Decompose the weight vectors.
+   *
+   * @tparam VecType The type of the vector used in the decommposition.
+   * @param weight The weight vector corresponding to a subproblem.
+   * @param idealPoint The reference point in the objective space.
+   * @param candidateFitness The objective vector of the candidate.
+   */
+  template<typename VecType>
+  typename VecType::elem_type Apply(const VecType& weight,
+                                    const VecType& idealPoint,
+                                    const VecType& candidateFitness)
+  {
+    return arma::max(weight % arma::abs(candidateFitness - idealPoint));
+  }
+};
+
+} // namespace ens
+
+#endif
@@ -0,0 +1,62 @@
+/**
+ * @file weighted_decomposition.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * The Weighted Average decomposition policy.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOEAD_WEIGHTED_HPP
+#define ENSMALLEN_MOEAD_WEIGHTED_HPP
+
+namespace ens {
+
+/**
+ * The Weighted average method of decomposition. The working principle is to
+ * minimize the dot product between reference direction and the line connecting
+ * objective vector and ideal point.
+ *
+ * For more information, see the following:
+ * @code
+ * article{zhang2007moea,
+ *   title={MOEA/D: A multiobjective evolutionary algorithm based on decomposition},
+ *   author={Zhang, Qingfu and Li, Hui},
+ *   journal={IEEE Transactions on evolutionary computation},
+ *   pages={712--731},
+ *   year={2007}
+ * @endcode
+ */
+class WeightedAverage
+{
+ public:
+  /**
+   * Constructor for Weighted Average decomposition policy.
+   */
+  WeightedAverage()
+  {
+    /* Nothing to do. */
+  }
+
+  /**
+   * Decompose the weight vectors.
+   *
+   * @tparam VecType The type of the vector used in the decommposition.
+   * @param weight The weight vector corresponding to a subproblem.
+   * @param idealPoint The reference point in the objective space.
+   * @param candidateFitness The objective vector of the candidate.
+   */
+  template<typename VecType>
+  typename VecType::elem_type Apply(const VecType& weight,
+                                    const VecType& /* idealPoint */,
+                                    const VecType& candidateFitness)
+  {
+    return arma::dot(weight, candidateFitness);
+  }
+};
+
+} // namespace ens
+
+#endif
@@ -0,0 +1,338 @@
+/**
+ * @file moead.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * MOEA/D-DE is a multi objective optimization algorithm. MOEA/D-DE
+ * uses genetic algorithms along with a set of reference directions
+ * to drive the population towards the Optimal Front.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_MOEAD_MOEAD_HPP
+#define ENSMALLEN_MOEAD_MOEAD_HPP
+
+//! Decomposition policies.
+#include "decomposition_policies/tchebycheff_decomposition.hpp"
+#include "decomposition_policies/weighted_decomposition.hpp"
+#include "decomposition_policies/pbi_decomposition.hpp"
+
+//! Weight initialization policies.
+#include "weight_init_policies/uniform_init.hpp"
+#include "weight_init_policies/bbs_init.hpp"
+#include "weight_init_policies/dirichlet_init.hpp"
+
+namespace ens {
+
+/**
+ * MOEA/D-DE (Multi Objective Evolutionary Algorithm based on Decompositon - 
+ * Differential Variant) is a multiobjective optimization algorithm. This class 
+ * implements the said optimizer. 
+ *
+ * The algorithm works by generating a candidate population from a fixed starting point. 
+ * Reference directions are generated to guide the optimization process towards the Pareto Front. 
+ * Further, a decomposition function is defined to decompose the problem to a scalar optimization 
+ * objective. Utilizing genetic operators, offsprings are generated with better decomposition values 
+ * to replace the neighboring parent solutions. 
+ *
+ * For more information, see the following:
+ * @code
+ * @article{li2008multiobjective,
+ *   title={Multiobjective optimization problems with complicated Pareto sets, MOEA/D and NSGA-II},
+ *   author={Li, Hui and Zhang, Qingfu},
+ *   journal={IEEE transactions on evolutionary computation},
+ *   pages={284--302},
+ *   year={2008},
+ * @endcode
+ */
+template<typename InitPolicyType = Uniform,
+         typename DecompPolicyType = Tchebycheff>
+class MOEAD {
+ public:
+  /**
+   * Constructor for the MOEA/D optimizer.
+   *
+   * The default values provided here are not necessarily suitable for a
+   * given function. Therefore, it is highly recommended to adjust the
+   * parameters according to the problem.
+   *
+   * @param populationSize The number of elements in the population.
+   * @param maxGenerations The maximum number of generations allowed.
+   * @param crossoverProb The probability that a crossover will occur.
+   * @param neighborProb The probability of sampling from neighbor.
+   * @param neighborSize The number of nearest neighbours of weights
+   *    to find.
+   * @param distributionIndex The crowding degree of the mutation.
+   * @param differentialWeight A parameter used in the mutation of candidate
+   *     solutions controls amplification factor of the differentiation.
+   * @param maxReplace The limit of solutions allowed to be replaced by a child.
+   * @param epsilon Handle numerical stability after weight initialization.
+   * @param lowerBound The lower bound on each variable of a member
+   *    of the variable space.
+   * @param upperBound The upper bound on each variable of a member
+   *    of the variable space.
+   */
+  MOEAD(const size_t populationSize = 300,
+        const size_t maxGenerations = 500,
+        const double crossoverProb = 1.0,
+        const double neighborProb = 0.9,
+        const size_t neighborSize = 20,
+        const double distributionIndex = 20,
+        const double differentialWeight = 0.5,
+        const size_t maxReplace = 2,
+        const double epsilon = 1E-10,
+        const arma::vec& lowerBound = arma::zeros(1, 1),
+        const arma::vec& upperBound = arma::ones(1, 1),
+        const InitPolicyType initPolicy = InitPolicyType(),
+        const DecompPolicyType decompPolicy = DecompPolicyType());
+
+  /**
+   * Constructor for the MOEA/D optimizer. This constructor is provides an
+   * overload to use lowerBound and upperBound as doubles, in case all the
+   * variables in the problem have the same limits.
+   *
+   * The default values provided here are not necessarily suitable for a
+   * given function. Therefore, it is highly recommended to adjust the
+   * parameters according to the problem.
+   *
+   * @param populationSize The number of elements in the population.
+   * @param maxGenerations The maximum number of generations allowed.
+   * @param crossoverProb The probability that a crossover will occur.
+   * @param neighborProb The probability of sampling from neighbor.
+   * @param neighborSize The number of nearest neighbours of weights
+   *    to find.
+   * @param distributionIndex The crowding degree of the mutation.
+   * @param differentialWeight A parameter used in the mutation of candidate
+   *     solutions controls amplification factor of the differentiation.
+   * @param maxReplace The limit of solutions allowed to be replaced by a child.
+   * @param epsilon Handle numerical stability after weight initialization.
+   * @param lowerBound The lower bound on each variable of a member
+   *    of the variable space.
+   * @param upperBound The upper bound on each variable of a member
+   *    of the variable space.
+   */
+    MOEAD(const size_t populationSize = 300,
+          const size_t maxGenerations = 500,
+          const double crossoverProb = 1.0,
+          const double neighborProb = 0.9,
+          const size_t neighborSize = 20,
+          const double distributionIndex = 20,
+          const double differentialWeight = 0.5,
+          const size_t maxReplace = 2,
+          const double epsilon = 1E-10,
+          const double lowerBound = 0,
+          const double upperBound = 1,
+          const InitPolicyType initPolicy = InitPolicyType(),
+          const DecompPolicyType decompPolicy = DecompPolicyType());
+
+  /**
+   * Optimize a set of objectives. The initial population is generated
+   * using the initial point. The output is the best generated front.
+   *
+   * @tparam MatType The type of matrix used to store coordinates.
+   * @tparam ArbitraryFunctionType The type of objective function.
+   * @tparam CallbackTypes Types of callback function.
+   * @param objectives std::tuple of the objective functions.
+   * @param iterate The initial reference point for generating population.
+   * @param callbacks The callback functions.
+   */
+  template<typename MatType,
+           typename... ArbitraryFunctionType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(std::tuple<ArbitraryFunctionType...>& objectives,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks);
+
+  //! Retrieve population size.
+  size_t PopulationSize() const { return populationSize; }
+  //! Modify the population size.
+  size_t& PopulationSize() { return populationSize; }
+
+  //! Retrieve number of generations.
+  size_t MaxGenerations() const { return maxGenerations; }
+  //! Modify the number of generations.
+  size_t& MaxGenerations() { return maxGenerations; }
+
+  //! Retrieve crossover rate.
+  double CrossoverRate() const { return crossoverProb; }
+  //! Modify the crossover rate.
+  double& CrossoverRate() { return crossoverProb; }
+
+  //! Retrieve size of the weight neighbor.
+  size_t NeighborSize() const { return neighborSize; }
+  //! Modify the size of the weight neighbor.
+  size_t& NeighborSize() { return neighborSize; }
+
+  //! Retrieve value of the distribution index.
+  double DistributionIndex() const { return distributionIndex; }
+  //! Modify the value of the distribution index.
+  double& DistributionIndex() { return distributionIndex; }
+
+  //! Retrieve value of neighbor probability.
+  double NeighborProb() const { return neighborProb; }
+  //! Modify the value of neigbourhood probability.
+  double& NeighborProb() { return neighborProb; }
+
+  //! Retrieve value of scaling factor.
+  double DifferentialWeight() const { return differentialWeight; }
+  //! Modify the value of scaling factor.
+  double& DifferentialWeight() { return differentialWeight; }
+
+  //! Retrieve value of maxReplace.
+  size_t MaxReplace() const { return maxReplace; }
+  //! Modify value of maxReplace.
+  size_t& MaxReplace() { return maxReplace; }
+
+   //! Retrieve value of epsilon.
+  double Epsilon() const { return epsilon; }
+  //! Modify value of maxReplace.
+  double& Epsilon() { return epsilon; }
+
+  //! Retrieve value of lowerBound.
+  const arma::vec& LowerBound() const { return lowerBound; }
+  //! Modify value of lowerBound.
+  arma::vec& LowerBound() { return lowerBound; }
+
+  //! Retrieve value of upperBound.
+  const arma::vec& UpperBound() const { return upperBound; }
+  //! Modify value of upperBound.
+  arma::vec& UpperBound() { return upperBound; }
+
+  //! Retrieve the Pareto optimal points in variable space. This returns an empty cube
+  //! until `Optimize()` has been called.
+  const arma::cube& ParetoSet() const { return paretoSet; }
+
+  //! Retrieve the best front (the Pareto frontier). This returns an empty cube until
+  //! `Optimize()` has been called.
+  const arma::cube& ParetoFront() const { return paretoFront; }
+
+  //! Get the weight initialization policy.
+  const InitPolicyType& InitPolicy() const { return initPolicy; }
+  //! Modify the weight initialization policy.
+  InitPolicyType& InitPolicy() { return initPolicy; }
+
+  //! Get the weight decomposition policy.
+  const DecompPolicyType& DecompPolicy() const { return decompPolicy; }
+  //! Modify the weight decomposition policy.
+  DecompPolicyType& DecompPolicy() { return decompPolicy; }
+
+ private:
+  /**
+   * @brief Randomly selects two members from the population.
+   *
+   * @param subProblemIdx Index of the current subproblem.
+   * @param neighborSize A matrix containing indices of the neighbors.
+   * @return std::tuple<size_t, size_t> The chosen pair of indices.
+   */
+  std::tuple<size_t, size_t> Mating(size_t subProblemIdx,
+                                    const arma::umat& neighborSize,
+                                    bool sampleNeighbor);
+
+  /**
+   * Mutate the child formed by the crossover of two random members of the
+   * population. Uses polynomial mutation.
+   *
+   * @tparam MatType The type of matrix used to store coordinates.
+   * @param child The candidate to be mutated.
+   * @param mutationRate The probability of mutation.
+   * @param lowerBound The lower bound on each variable in the matrix.
+   * @param upperBound The upper bound on each variable in the matrix.
+   * @return The mutated child.
+   */
+  template<typename MatType>
+  void Mutate(MatType& child,
+              double mutationRate,
+              const MatType& lowerBound,
+              const MatType& upperBound);
+
+  /**
+   * Evaluate objectives for the elite population.
+   *
+   * @tparam ArbitraryFunctionType std::tuple of multiple function types.
+   * @tparam MatType Type of matrix to optimize.
+   * @param population The elite population.
+   * @param objectives The set of objectives.
+   * @param calculatedObjectives Vector to store calculated objectives.
+   */
+  template<std::size_t I = 0,
+           typename MatType,
+           typename ...ArbitraryFunctionType>
+  typename std::enable_if<I == sizeof...(ArbitraryFunctionType), void>::type
+  EvaluateObjectives(
+                     std::vector<MatType>&,
+                     std::tuple<ArbitraryFunctionType...>&,
+                     std::vector<arma::Col<typename MatType::elem_type> >&);
+
+  template<std::size_t I = 0,
+           typename MatType,
+           typename ...ArbitraryFunctionType>
+  typename std::enable_if<I < sizeof...(ArbitraryFunctionType), void>::type
+  EvaluateObjectives(
+                     std::vector<MatType>& population,
+                     std::tuple<ArbitraryFunctionType...>& objectives,
+                     std::vector<arma::Col<typename MatType::elem_type> >&
+                     calculatedObjectives);
+
+  //! Size of the population.
+  size_t populationSize;
+
+  //! Maximum number of generations before termination criteria is met.
+  size_t maxGenerations;
+
+  //! Probability of crossover between two members.
+  double crossoverProb;
+
+  //! The probability that two elements will be chosen from the neighbor.
+  double neighborProb;
+
+  //! Number of nearest neighbours of weights to consider.
+  size_t neighborSize;
+
+  //! The crowding degree of the mutation. Higher value produces a mutant
+  //! resembling its parent.
+  double distributionIndex;
+
+  //! Amplification factor for differentiation.
+  double differentialWeight;
+
+  //! Maximum number of childs which can replace the parent. Higher value
+  //! leads to a loss of diversity.
+  size_t maxReplace;
+
+  //! A small numeric value to be added to the weights after initialization.
+  //! Prevents zero value inside inited weights.
+  double epsilon;
+
+  //! Lower bound on each variable in the variable space.
+  arma::vec lowerBound;
+
+  //! Upper bound on each variable in the variable space.
+  arma::vec upperBound;
+
+  //! The set of all the Pareto optimal points.
+  //! Stored after Optimize() is called.
+  arma::cube paretoSet;
+
+  //! The set of all the Pareto optimal objective vectors.
+  //! Stored after Optimize() is called.
+  arma::cube paretoFront;
+
+  //! Policy to initialize the reference directions (weights) matrix.
+  InitPolicyType initPolicy;
+
+  //! Policy to decompose the weights.
+  DecompPolicyType decompPolicy;
+};
+
+using DefaultMOEAD = MOEAD<Uniform, Tchebycheff>;
+using BBSMOEAD = MOEAD<BayesianBootstrap, Tchebycheff>;
+using DirichletMOEAD = MOEAD<Dirichlet, Tchebycheff>;
+} // namespace ens
+
+// Include implementation.
+#include "moead_impl.hpp"
+
+#endif
@@ -0,0 +1,441 @@
+/**
+ * @file moead_impl.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * Implementation of the MOEA/D-DE algorithm. Used for multi-objective
+ * optimization problems on arbitrary functions.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more Information.
+ */
+
+#ifndef ENSMALLEN_MOEAD_MOEAD_IMPL_HPP
+#define ENSMALLEN_MOEAD_MOEAD_IMPL_HPP
+
+#include "moead.hpp"
+#include <assert.h>
+
+namespace ens {
+template <typename InitPolicyType, typename DecompPolicyType>
+inline MOEAD<InitPolicyType, DecompPolicyType>::
+MOEAD(const size_t populationSize,
+      const size_t maxGenerations,
+      const double crossoverProb,
+      const double neighborProb,
+      const size_t neighborSize,
+      const double distributionIndex,
+      const double differentialWeight,
+      const size_t maxReplace,
+      const double epsilon,
+      const arma::vec& lowerBound,
+      const arma::vec& upperBound,
+      const InitPolicyType initPolicy,
+      const DecompPolicyType decompPolicy) :
+    populationSize(populationSize),
+    maxGenerations(maxGenerations),
+    crossoverProb(crossoverProb),
+    neighborProb(neighborProb),
+    neighborSize(neighborSize),
+    distributionIndex(distributionIndex),
+    differentialWeight(differentialWeight),
+    maxReplace(maxReplace),
+    epsilon(epsilon),
+    lowerBound(lowerBound),
+    upperBound(upperBound),
+    initPolicy(initPolicy),
+    decompPolicy(decompPolicy)
+  { /* Nothing to do here. */ }
+
+template <typename InitPolicyType, typename DecompPolicyType>
+inline MOEAD<InitPolicyType, DecompPolicyType>::
+MOEAD(const size_t populationSize,
+      const size_t maxGenerations,
+      const double crossoverProb,
+      const double neighborProb,
+      const size_t neighborSize,
+      const double distributionIndex,
+      const double differentialWeight,
+      const size_t maxReplace,
+      const double epsilon,
+      const double lowerBound,
+      const double upperBound,
+      const InitPolicyType initPolicy,
+      const DecompPolicyType decompPolicy) :
+    populationSize(populationSize),
+    maxGenerations(maxGenerations),
+    crossoverProb(crossoverProb),
+    neighborProb(neighborProb),
+    neighborSize(neighborSize),
+    distributionIndex(distributionIndex),
+    differentialWeight(differentialWeight),
+    maxReplace(maxReplace),
+    epsilon(epsilon),
+    lowerBound(lowerBound * arma::ones(1, 1)),
+    upperBound(upperBound * arma::ones(1, 1)),
+    initPolicy(initPolicy),
+    decompPolicy(decompPolicy)
+  { /* Nothing to do here. */ }
+
+//! Optimize the function.
+template <typename InitPolicyType, typename DecompPolicyType>
+template<typename MatType,
+         typename... ArbitraryFunctionType,
+         typename... CallbackTypes>
+typename MatType::elem_type MOEAD<InitPolicyType, DecompPolicyType>::
+Optimize(std::tuple<ArbitraryFunctionType...>& objectives,
+         MatType& iterateIn,
+         CallbackTypes&&... callbacks)
+{
+  // Population Size must be at least 3 for MOEA/D-DE to work.
+  if (populationSize < 3)
+  {
+    throw std::logic_error("MOEA/D-DE::Optimize(): population size should be at least"
+        " 3!");
+  }
+
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  // Make sure that we have the methods that we need.  Long name...
+  traits::CheckArbitraryFunctionTypeAPI<ArbitraryFunctionType...,
+      BaseMatType>();
+  RequireDenseFloatingPointType<BaseMatType>();
+
+  if (neighborSize < 2)
+  {
+    throw std::invalid_argument(
+        "neighborSize should be atleast 2, however "
+        + std::to_string(neighborSize) + " was detected."
+    );
+  }
+
+  if (neighborSize > populationSize - 1u)
+  {
+    std::ostringstream oss;
+    oss << "MOEAD::Optimize(): " << "neighborSize is " << neighborSize
+        << " but populationSize is " << populationSize << "(should be"
+        << " atleast " << (neighborSize + 1u) << ")" << std::endl;
+    throw std::logic_error(oss.str());
+  }
+
+  // Check if lower bound is a vector of a single dimension.
+  if (lowerBound.n_rows == 1)
+    lowerBound = lowerBound(0, 0) * arma::ones(iterate.n_rows, iterate.n_cols);
+
+  // Check if upper bound is a vector of a single dimension.
+  if (upperBound.n_rows == 1)
+    upperBound = upperBound(0, 0) * arma::ones(iterate.n_rows, iterate.n_cols);
+
+  // Check the dimensions of lowerBound and upperBound.
+  assert(lowerBound.n_rows == iterate.n_rows && "The dimensions of "
+      "lowerBound are not the same as the dimensions of iterate.");
+  assert(upperBound.n_rows == iterate.n_rows && "The dimensions of "
+      "upperBound are not the same as the dimensions of iterate.");
+
+  const size_t numObjectives = sizeof...(ArbitraryFunctionType);
+  const size_t numVariables = iterate.n_rows;
+
+  //! Useful temporaries for float-like comparisons.
+  const BaseMatType castedLowerBound = arma::conv_to<BaseMatType>::from(lowerBound);
+  const BaseMatType castedUpperBound = arma::conv_to<BaseMatType>::from(upperBound);
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
+  // The weight matrix. Each vector represents a decomposition subproblem (M X N).
+  const BaseMatType weights = initPolicy.template Generate<BaseMatType>(
+      numObjectives, populationSize, epsilon);
+
+  // 1.1 Storing the indices of nearest neighbors of each weight vector.
+  arma::umat neighborIndices(neighborSize, populationSize);
+  for (size_t i = 0; i < populationSize; ++i)
+  {
+    // Cache the distance between weights[i] and other weights.
+    const arma::Row<ElemType> distances =
+        arma::sqrt(arma::sum(arma::square(weights.col(i) - weights.each_col())));
+    arma::uvec sortedIndices = arma::stable_sort_index(distances);
+    // Ignore distance from self.
+    neighborIndices.col(i) = sortedIndices(arma::span(1, neighborSize));
+  }
+
+  // 1.2 Random generation of the initial population.
+  std::vector<BaseMatType> population(populationSize);
+  for (BaseMatType& individual : population)
+  {
+    individual = arma::randu<BaseMatType>(
+        iterate.n_rows, iterate.n_cols) - 0.5 + iterate;
+
+    // Constrain all genes to be within bounds.
+    individual = arma::min(arma::max(individual, castedLowerBound), castedUpperBound);
+  }
+
+  Info << "MOEA/D-DE initialized successfully. Optimization started." << std::endl;
+
+  std::vector<arma::Col<ElemType>> populationFitness(populationSize);
+  std::fill(populationFitness.begin(), populationFitness.end(),
+      arma::Col<ElemType>(numObjectives, arma::fill::zeros));
+  EvaluateObjectives(population, objectives, populationFitness);
+
+  // 1.3 Initialize the ideal point z.
+  arma::Col<ElemType> idealPoint(numObjectives);
+  idealPoint.fill(std::numeric_limits<ElemType>::max());
+
+  for (const arma::Col<ElemType>& individualFitness : populationFitness)
+    idealPoint = arma::min(idealPoint, individualFitness);
+
+  Callback::BeginOptimization(*this, objectives, iterate, callbacks...);
+
+  // 2 The main loop.
+  for (size_t generation = 1; generation <= maxGenerations && !terminate; ++generation)
+  {
+    // Shuffle indexes of subproblems.
+    const arma::uvec shuffle = arma::shuffle(
+        arma::linspace<arma::uvec>(0, populationSize - 1, populationSize));
+    for (size_t subProblemIdx : shuffle)
+    {
+      // 2.1 Randomly select two indices in neighborIndices[subProblemIdx] and use them
+      // to make a child.
+      size_t r1, r2, r3;
+      r1 = subProblemIdx;
+      // Randomly choose to sample from the population or the neighbors.
+      const bool sampleNeighbor = arma::randu() < neighborProb;
+      std::tie(r2, r3) =
+          Mating(subProblemIdx, neighborIndices, sampleNeighbor);
+
+      // 2.2 - 2.3 Reproduction and Repair: Differential Operator followed by
+      // Polynomial Mutation.
+      BaseMatType candidate(iterate.n_rows, iterate.n_cols);
+
+      for (size_t geneIdx = 0; geneIdx < numVariables; ++geneIdx)
+      {
+        if (arma::randu() < crossoverProb)
+        {
+          candidate(geneIdx) = population[r1](geneIdx) +
+              differentialWeight * (population[r2](geneIdx) -
+                  population[r3](geneIdx));
+
+          // Boundary conditions.
+          if (candidate(geneIdx) < castedLowerBound(geneIdx))
+          {
+            candidate(geneIdx) = castedLowerBound(geneIdx) +
+                arma::randu() * (population[r1](geneIdx) - castedLowerBound(geneIdx));
+          }
+          if (candidate(geneIdx) > castedUpperBound(geneIdx))
+          {
+            candidate(geneIdx) = castedUpperBound(geneIdx) -
+                arma::randu() * (castedUpperBound(geneIdx) - population[r1](geneIdx));
+          }
+        }
+        else
+          candidate(geneIdx) = population[r1](geneIdx);
+      }
+
+      Mutate(candidate, 1.0 / static_cast<double>(numVariables),
+          castedLowerBound, castedUpperBound);
+
+      arma::Col<ElemType> candidateFitness(numObjectives);
+      //! Creating temp vectors to pass to EvaluateObjectives.
+      std::vector<BaseMatType> candidateContainer { candidate };
+      std::vector<arma::Col<ElemType>> fitnessContainer { candidateFitness };
+      EvaluateObjectives(candidateContainer, objectives, fitnessContainer);
+      candidateFitness = std::move(fitnessContainer[0]);
+      //! Flush out the dummy containers.
+      fitnessContainer.clear();
+      candidateContainer.clear();
+
+      // 2.4 Update of ideal point.
+      idealPoint = arma::min(idealPoint, candidateFitness);
+
+      // 2.5 Update of the population.
+      size_t replaceCounter = 0;
+      const size_t sampleSize = sampleNeighbor ? neighborSize : populationSize;
+
+      const arma::uvec idxShuffle = arma::shuffle(
+          arma::linspace<arma::uvec>(0, sampleSize - 1, sampleSize));
+
+      for (size_t idx : idxShuffle)
+      {
+        // Preserve diversity by controlling replacement of neighbors
+        // by child solution.
+        if (replaceCounter >= maxReplace)
+          break;
+
+        const size_t pick = sampleNeighbor ?
+            neighborIndices(idx, subProblemIdx) : idx;
+
+        const ElemType candidateDecomposition = decompPolicy.template
+            Apply<arma::Col<ElemType>>(weights.col(pick), idealPoint, candidateFitness);
+        const ElemType parentDecomposition =  decompPolicy.template
+            Apply<arma::Col<ElemType>>(weights.col(pick), idealPoint, populationFitness[pick]);
+
+        if (candidateDecomposition < parentDecomposition)
+        {
+          population[pick] = candidate;
+          populationFitness[pick] = candidateFitness;
+          ++replaceCounter;
+        }
+      }
+    } // End of pass over all subproblems.
+
+    //  The final population itself is the best front.
+    const std::vector<arma::uvec> frontIndices { arma::shuffle(
+        arma::linspace<arma::uvec>(0, populationSize - 1, populationSize)) };
+
+    terminate |= Callback::GenerationalStepTaken(*this, objectives, iterate,
+        populationFitness, frontIndices, callbacks...);
+  } // End of pass over all the generations.
+
+  // Set the candidates from the Pareto Set as the output.
+  paretoSet.set_size(population[0].n_rows, population[0].n_cols, population.size());
+
+  // The Pareto Front is stored, can be obtained via ParetoSet() getter.
+  for (size_t solutionIdx = 0; solutionIdx < population.size(); ++solutionIdx)
+  {
+    paretoSet.slice(solutionIdx) =
+        arma::conv_to<arma::mat>::from(population[solutionIdx]);
+  }
+
+  // Set the candidates from the Pareto Front as the output.
+  paretoFront.set_size(populationFitness[0].n_rows, populationFitness[0].n_cols,
+      populationFitness.size());
+
+  // The Pareto Front is stored, can be obtained via ParetoFront() getter.
+  for (size_t solutionIdx = 0; solutionIdx < populationFitness.size(); ++solutionIdx)
+  {
+    paretoFront.slice(solutionIdx) =
+        arma::conv_to<arma::mat>::from(populationFitness[solutionIdx]);
+  }
+
+  // Assign iterate to first element of the Pareto Set.
+  iterate = population[0];
+
+  Callback::EndOptimization(*this, objectives, iterate, callbacks...);
+
+  ElemType performance = std::numeric_limits<ElemType>::max();
+
+  for (size_t geneIdx = 0; geneIdx < numObjectives; ++geneIdx)
+  {
+    if (arma::accu(populationFitness[geneIdx]) < performance)
+      performance = arma::accu(populationFitness[geneIdx]);
+  }
+
+  return performance;
+}
+
+//! Randomly chooses to select from parents or neighbors.
+template <typename InitPolicyType, typename DecompPolicyType>
+inline std::tuple<size_t, size_t>
+MOEAD<InitPolicyType, DecompPolicyType>::
+Mating(size_t subProblemIdx,
+       const arma::umat& neighborIndices,
+       bool sampleNeighbor)
+{
+  //! Indexes of two points from the sample space.
+  size_t pointA = sampleNeighbor
+      ? neighborIndices(
+            arma::randi(arma::distr_param(0, neighborSize - 1u)), subProblemIdx)
+      : arma::randi(arma::distr_param(0, populationSize - 1u));
+
+  size_t pointB = sampleNeighbor
+      ? neighborIndices(
+            arma::randi(arma::distr_param(0, neighborSize - 1u)), subProblemIdx)
+      : arma::randi(arma::distr_param(0, populationSize - 1u));
+
+  //! If the sampled points are equal, then modify one of them
+  //! within reasonable bounds.
+  if (pointA == pointB)
+  {
+    if (pointA == populationSize - 1u)
+      --pointA;
+    else
+      ++pointA;
+  }
+
+  return std::make_tuple(pointA, pointB);
+}
+
+//! Perform Polynomial mutation of the candidate.
+template <typename InitPolicyType, typename DecompPolicyType>
+template<typename MatType>
+inline void MOEAD<InitPolicyType, DecompPolicyType>::
+Mutate(MatType& candidate,
+       double mutationRate,
+       const MatType& lowerBound,
+       const MatType& upperBound)
+{
+    const size_t numVariables = candidate.n_rows;
+    for (size_t geneIdx = 0; geneIdx < numVariables; ++geneIdx)
+    {
+      // Should this gene be mutated?
+      if (arma::randu() > mutationRate)
+        continue;
+
+      const double geneRange = upperBound(geneIdx) - lowerBound(geneIdx);
+      // Normalised distance from the bounds.
+      const double lowerDelta = (candidate(geneIdx) - lowerBound(geneIdx)) / geneRange;
+      const double upperDelta = (upperBound(geneIdx) - candidate(geneIdx)) / geneRange;
+      const double mutationPower = 1. / (distributionIndex + 1.0);
+      const double rand = arma::randu();
+      double value, perturbationFactor;
+      if (rand < 0.5)
+      {
+        value = 2.0 * rand + (1.0 - 2.0 * rand) *
+            std::pow(upperDelta, distributionIndex + 1.0);
+        perturbationFactor = std::pow(value, mutationPower) - 1.0;
+      }
+      else
+      {
+        value = 2.0 * (1.0 - rand) + 2.0 *(rand - 0.5) *
+            std::pow(lowerDelta, distributionIndex + 1.0);
+        perturbationFactor = 1.0 - std::pow(value, mutationPower);
+      }
+
+      candidate(geneIdx) += perturbationFactor * geneRange;
+    }
+    //! Enforce bounds.
+    candidate = arma::min(arma::max(candidate, lowerBound), upperBound);
+}
+
+//! No objectives to evaluate.
+template <typename InitPolicyType, typename DecompPolicyType>
+template<std::size_t I,
+         typename MatType,
+         typename ...ArbitraryFunctionType>
+typename std::enable_if<I == sizeof...(ArbitraryFunctionType), void>::type
+MOEAD<InitPolicyType, DecompPolicyType>::
+EvaluateObjectives(
+    std::vector<MatType>&,
+    std::tuple<ArbitraryFunctionType...>&,
+    std::vector<arma::Col<typename MatType::elem_type> >&)
+{
+ // Nothing to do here.
+}
+
+//! Evaluate the objectives for the entire population.
+template <typename InitPolicyType, typename DecompPolicyType>
+template<std::size_t I,
+         typename MatType,
+         typename ...ArbitraryFunctionType>
+typename std::enable_if<I < sizeof...(ArbitraryFunctionType), void>::type
+MOEAD<InitPolicyType, DecompPolicyType>::
+EvaluateObjectives(
+    std::vector<MatType>& population,
+    std::tuple<ArbitraryFunctionType...>& objectives,
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives)
+{
+  for (size_t i = 0; i < population.size(); i++)
+  {
+    calculatedObjectives[i](I) = std::get<I>(objectives).Evaluate(population[i]);
+    EvaluateObjectives<I+1, MatType, ArbitraryFunctionType...>(population, objectives,
+                                                               calculatedObjectives);
+  }
+}
+
+}  // namespace ens
+
+#endif
@@ -0,0 +1,76 @@
+/**
+ * @file bbs_init.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * The Bayesian Bootstrap (BBS) method of Weight Initialization.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOEAD_BBS_HPP
+#define ENSMALLEN_MOEAD_BBS_HPP
+
+namespace ens {
+
+/**
+ * The Bayesian Bootstrap method for initializing weights. Samples are randomly picked from uniform 
+ * distribution followed by sorting and finding adjacent difference. This gives you a list of 
+ * numbers which is guaranteed to sum up to 1.
+ *
+ * @code
+ * @article{rubin1981bayesian,
+ *   title={The bayesian bootstrap},
+ *   author={Rubin, Donald B},
+ *   journal={The annals of statistics},
+ *   pages={130--134},
+ *   year={1981},
+ * @endcode
+ *
+ */
+class BayesianBootstrap
+{
+ public:
+  /**
+   * Constructor for Bayesian Bootstrap policy.
+   */
+  BayesianBootstrap()
+  {
+    /* Nothing to do. */
+  }
+
+  /**
+   * Generate the reference direction matrix.
+   *
+   * @tparam MatType The type of the matrix used for constructing weights.
+   * @param numObjectives The dimensionality of objective space.
+   * @param numPoints The number of reference directions requested.
+   * @param epsilon Handle numerical stability after weight initialization.
+   */
+  template<typename MatType>
+  MatType Generate(const size_t numObjectives,
+                   const size_t numPoints,
+                   const double epsilon)
+  {
+      typedef typename MatType::elem_type ElemType;
+      typedef typename arma::Col<ElemType> VecType;
+
+      MatType weights(numObjectives, numPoints);
+      for (size_t pointIdx = 0; pointIdx < numPoints; ++pointIdx)
+      {
+        VecType referenceDirection(numObjectives + 1, arma::fill::randu);
+        referenceDirection(0) = 0;
+        referenceDirection(numObjectives) = 1;
+        referenceDirection = arma::sort(referenceDirection);
+        referenceDirection = arma::diff(referenceDirection);
+        weights.col(pointIdx) = std::move(referenceDirection) + epsilon;
+      }
+
+      return weights;
+  }
+};
+
+} // namespace ens
+
+#endif
@@ -0,0 +1,55 @@
+/**
+ * @file dirichlet_init.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * The Dirichlet method of Weight Initialization.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOEAD_DIRICHLET_HPP
+#define ENSMALLEN_MOEAD_DIRICHLET_HPP
+
+namespace ens {
+
+/**
+ * The Dirichlet method for initializing weights. Sampling a 
+ * Dirichlet distribution with parameters set to one returns 
+ * point lying on unit simplex with uniform distribution.
+ */
+class Dirichlet
+{
+ public:
+  /**
+   * Constructor for Dirichlet policy.
+   */
+  Dirichlet()
+  {
+    /* Nothing to do. */
+  }
+
+  /**
+   * Generate the reference direction matrix.
+   *
+   * @tparam MatType The type of the matrix used for constructing weights.
+   * @param numObjectives The dimensionality of objective space.
+   * @param numPoints The number of reference directions requested.
+   * @param epsilon Handle numerical stability after weight initialization.
+   */
+  template<typename MatType>
+  MatType Generate(const size_t numObjectives,
+                   const size_t numPoints,
+                   const double epsilon)
+  {
+    MatType weights = arma::randg<MatType>(numObjectives, numPoints,
+        arma::distr_param(1.0, 1.0)) + epsilon;
+    // Normalize each column.
+    return arma::normalise(weights, 1, 0);
+  }
+};
+
+} // namespace ens
+
+#endif
@@ -0,0 +1,208 @@
+/**
+ * @file uniform_init.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * The Uniform (Das Dennis) methodology of Weight Initialization.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOEAD_UNIFORM_HPP
+#define ENSMALLEN_MOEAD_UNIFORM_HPP
+
+namespace ens {
+
+/**
+ * The Uniform (Das Dennis) method for initializing weights. This algorithm guarantees
+ * that the distance between adjacent points would be uniform.
+ *
+ * For more information, see the following:
+ *
+ * @code
+ * article{zhang2007moea,
+ *   title={MOEA/D: A multiobjective evolutionary algorithm based on decomposition},
+ *   author={Zhang, Qingfu and Li, Hui},
+ *   journal={IEEE Transactions on evolutionary computation},
+ *   pages={712--731},
+ *   year={2007}
+ * @endcode
+ */
+class Uniform
+{
+ public:
+  /**
+   * Constructor for Uniform Weight Initializatoin Policy.
+   */
+  Uniform()
+  {
+    /* Nothing to do. */
+  }
+
+  /**
+   * Generate the reference direction matrix.
+   *
+   * @tparam MatType The type of the matrix used for constructing weights.
+   * @param numObjectives The dimensionality of objective space.
+   * @param numPoints The number of reference directions requested.
+   * @param epsilon Handle numerical stability after weight initialization.
+   */
+  template<typename MatType>
+  MatType Generate(size_t numObjectives,
+                   size_t numPoints,
+                   double epsilon)
+  {
+    size_t numPartitions  = FindNumParitions(numObjectives, numPoints);
+    size_t validNumPoints = FindNumUniformPoints(numObjectives, numPartitions);
+
+    //! The requested number of points is not matching any partition number.
+    if (numPoints != validNumPoints)
+    {
+      size_t nextValidNumPoints = FindNumUniformPoints(numObjectives, numPartitions + 1);
+      std::ostringstream oss;
+      oss << "DasDennis::Generate(): " << "The requested numPoints " << numPoints
+          << " cannot be generated uniformly.\n " << "Either choose numPoints as "
+          << validNumPoints << " (numPartition = " << numPartitions << ") or "
+          << "numPoints as " << nextValidNumPoints << " (numPartition = "
+          << numPartitions + 1 << ").";
+      throw std::logic_error(oss.str());
+    }
+
+    return DasDennis<MatType>(numObjectives, numPoints,
+        numPartitions, epsilon);
+  }
+
+ private:
+  /**
+   * Finds the number of points which can be sampled uniformly from a 
+   * unit simplex given the number of partitions.
+   */
+  size_t FindNumUniformPoints(const size_t numObjectives,
+                              const size_t numPartitions)
+  {
+    //! O(N) algorithm to calculate binomial coefficient.
+    //! Source: https://www.geeksforgeeks.org/space-and-time-efficient-binomial-coefficient/
+    auto BinomialCoefficient =
+        [](size_t n, size_t k) -> size_t
+        {
+          size_t retval = 1;
+          // Since, C(n, k) = C(n, n - k).
+          if (k > n - k)
+            k = n - k;
+
+          // [n * (n - 1) * .... * (n - k + 1)] / [k * (k - 1) * .... * 1].
+          for (size_t i = 0; i < k; ++i)
+          {
+            retval *= (n - i);
+            retval /= (i + 1);
+          }
+
+          return retval;
+        };
+    return BinomialCoefficient(numObjectives + numPartitions - 1, numPartitions);
+  }
+
+  /**
+   *  Calculates the appropriate number of partitions such that, the binomial
+   *  coefficient value is closest to the number of points requested.
+   */
+  size_t FindNumParitions(size_t numObjectives, size_t numPoints)
+  {
+    if (numObjectives == 1) return 0;
+    // Iteratively increase numPartitions so that the binomial coefficient
+    // comes near to numPoints;
+    size_t numPartitions {1};
+    size_t sampledNumPoints = FindNumUniformPoints(numPartitions,
+        numObjectives);
+    while (sampledNumPoints <= numPoints)
+    {
+       ++numPartitions;
+       sampledNumPoints = FindNumUniformPoints(numObjectives,
+          numPartitions);
+    }
+
+    return numPartitions - 1;
+  }
+
+  /**
+   * A helper function for DasDennis
+   */
+  template<typename AuxInfoStackType,
+           typename MatType>
+  void DasDennisHelper(AuxInfoStackType& progressStack,
+                       MatType& weights,
+                       const size_t numObjectives,
+                       const size_t numPoints,
+                       const size_t numPartitions,
+                       const double epsilon)
+  {
+    typedef typename MatType::elem_type ElemType;
+    typedef typename arma::Row<ElemType> RowType;
+
+    size_t counter = 0;
+    const ElemType delta = 1.0 / (ElemType)numPartitions;
+
+    while ((counter < numPoints) && !progressStack.empty())
+    {
+      MatType point{};
+      size_t beta{};
+      std::tie(point, beta) = progressStack.back();
+      progressStack.pop_back();
+
+      if (point.size() + 1 == numObjectives)
+      {
+        point.insert_rows(point.n_rows, RowType(1).fill(
+            delta * static_cast<ElemType>(beta)));
+        weights.col(counter) = point + epsilon;
+        ++counter;
+      }
+
+      else
+      {
+        for (size_t i = 0; i <= beta; ++i)
+        {
+          MatType pointClone(point);
+          pointClone.insert_rows(pointClone.n_rows, RowType(1).fill(
+              delta * static_cast<ElemType>(i)));
+          progressStack.push_back({pointClone, beta - i});
+        }
+      }
+    }
+  }
+
+  /**
+   * Generates the weight matrix after verifying the
+   * validity of the parameters.
+   */
+  template <typename MatType>
+  MatType DasDennis(const size_t numObjectives,
+                    const size_t numPoints,
+                    const size_t numPartitions,
+                    const double epsilon)
+  {
+    //! Holds auxillary information required for the helper function. 
+    //! Holds the current point and beta value.
+    using AuxContainer = std::pair<MatType, size_t>;
+
+    std::vector<AuxContainer> progressStack{};
+    //! Init the progress stack.
+    progressStack.push_back({{}, numPartitions});
+    MatType weights(numObjectives, numPoints);
+    weights.fill(arma::datum::nan);
+    DasDennisHelper<decltype(progressStack), MatType>(
+        progressStack,
+        weights,
+        numObjectives,
+        numPoints,
+        numPartitions,
+        epsilon);
+
+    return weights;
+  }
+
+};
+
+} // namespace ens
+
+#endif
@@ -1,6 +1,7 @@
 /**
 * @file nsga2.hpp
 * @author Sayan Goswami
+ * @author Nanubala Gnana Sai
 *
 * NSGA-II is a multi-objective optimization algorithm, widely used in
 * many real-world applications. NSGA-II generates offsprings using
@@ -50,7 +51,8 @@ namespace ens {
 * see the documentation on function types included with this distribution or
 * on the ensmallen website.
 */
-class NSGA2 {
+class NSGA2
+{
 public:
  /**
   * Constructor for the NSGA2 optimizer.
@@ -168,9 +170,33 @@ class NSGA2 {
  //! Modify value of upperBound.
  arma::vec& UpperBound() { return upperBound; }

-  //! Retrieve the best front (the Pareto frontier).  This returns an empty vector until `Optimize()`
-  //! has been called.
-  const std::vector<arma::mat>& Front() const { return bestFront; }
+  //! Retrieve the Pareto optimal points in variable space. This returns an empty cube
+  //! until `Optimize()` has been called.
+  const arma::cube& ParetoSet() const { return paretoSet; }
+
+  //! Retrieve the best front (the Pareto frontier). This returns an empty cube until
+  //! `Optimize()` has been called.
+  const arma::cube& ParetoFront() const { return paretoFront; }
+
+  /**
+   * Retrieve the best front (the Pareto frontier).  This returns an empty
+   * vector until `Optimize()` has been called.  Note that this function is
+   * deprecated and will be removed in ensmallen 3.x!  Use `ParetoFront()`
+   * instead.
+   */
+  [[deprecated("use ParetoFront() instead")]] const std::vector<arma::mat>& Front()
+  {
+    if (rcFront.size() == 0)
+    {
+      // Match the old return format.
+      for (size_t i = 0; i < paretoFront.n_slices; ++i)
+      {
+        rcFront.push_back(arma::mat(paretoFront.slice(i)));
+      }
+    }
+
+    return rcFront;
+  }

 private:
  /**
@@ -188,7 +214,7 @@ class NSGA2 {
  typename std::enable_if<I == sizeof...(ArbitraryFunctionType), void>::type
  EvaluateObjectives(std::vector<MatType>&,
                     std::tuple<ArbitraryFunctionType...>&,
-                     std::vector<arma::Col<double> >&);
+                     std::vector<arma::Col<typename MatType::elem_type> >&);

  template<std::size_t I = 0,
           typename MatType,
@@ -196,7 +222,8 @@ class NSGA2 {
  typename std::enable_if<I < sizeof...(ArbitraryFunctionType), void>::type
  EvaluateObjectives(std::vector<MatType>& population,
                     std::tuple<ArbitraryFunctionType...>& objectives,
-                     std::vector<arma::Col<double> >& calculatedObjectives);
+                     std::vector<arma::Col<typename MatType::elem_type> >&
+                     calculatedObjectives);

  /**
   * Reproduce candidates from the elite population to generate a new
@@ -210,8 +237,8 @@ class NSGA2 {
   */
  template<typename MatType>
  void BinaryTournamentSelection(std::vector<MatType>& population,
-                                 const arma::vec& lowerBound,
-                                 const arma::vec& upperBound);
+                                 const MatType& lowerBound,
+                                 const MatType& upperBound);

  /**
   * Crossover two parents to create a pair of new children.
@@ -239,8 +266,8 @@ class NSGA2 {
   */
  template<typename MatType>
  void Mutate(MatType& child,
-              const arma::vec& lowerBound,
-              const arma::vec& upperBound);
+              const MatType& lowerBound,
+              const MatType& upperBound);

  /**
   * Sort the candidate population using their domination count and the set of
@@ -281,11 +308,15 @@ class NSGA2 {
   * Assigns crowding distance metric for sorting.
   *
   * @param front The previously generated Pareto fronts.
-   * @param objectives The set of objectives.
-   * @param crowdingDistance The previously calculated objectives.
+   * @param calculatedObjectives The previously calculated objectives.
+   * @param crowdingDistance The crowding distance for each individual in
+   *    the population.
   */
-  void CrowdingDistanceAssignment(const std::vector<size_t>& front,
-                                  std::vector<double>& crowdingDistance);
+  template <typename MatType>
+  void CrowdingDistanceAssignment(
+      const std::vector<size_t>& front,
+      std::vector<arma::Col<typename MatType::elem_type>>& calculatedObjectives,
+      std::vector<typename MatType::elem_type>& crowdingDistance);

  /**
   * The operator used in the crowding distance based sorting.
@@ -299,13 +330,15 @@ class NSGA2 {
   * @param idxQ The index of the second cadidate from the elite population
   *     being sorted.
   * @param ranks The previously calculated ranks.
-   * @param crowdingDistance The previously calculated objectives.
+   * @param crowdingDistance The crowding distance for each individual in
+   *    the population.
   * @return true if the first candidate is preferred, otherwise, false.
   */
+  template<typename MatType>
  bool CrowdingOperator(size_t idxP,
                        size_t idxQ,
                        const std::vector<size_t>& ranks,
-                        const std::vector<double>& crowdingDistance);
+                        const std::vector<typename MatType::elem_type>& crowdingDistance);

  //! The number of objectives being optimised for.
  size_t numObjectives;
@@ -337,8 +370,18 @@ class NSGA2 {
  //! Upper bound of the initial swarm.
  arma::vec upperBound;

-  //! Best front, stored after Optimize() is called.
-  std::vector<arma::mat> bestFront;
+  //! The set of all the Pareto optimal points.
+  //! Stored after Optimize() is called.
+  arma::cube paretoSet;
+
+  //! The set of all the Pareto optimal objective vectors.
+  //! Stored after Optimize() is called.
+  arma::cube paretoFront;
+
+  //! A different representation of the Pareto front, for reverse compatibility
+  //! purposes.  This can be removed when ensmallen 3.x is released!  (Along
+  //! with `Front()`.)  This is only populated when `Front()` is called.
+  std::vector<arma::mat> rcFront;
 };

 } // namespace ens
@@ -1,6 +1,7 @@
 /**
 * @file nsga2_impl.hpp
 * @author Sayan Goswami
+ * @author Nanubala Gnana Sai
 *
 * Implementation of the NSGA-II algorithm. Used for multi-objective
 * optimization problems on arbitrary functions.
@@ -27,6 +28,8 @@ inline NSGA2::NSGA2(const size_t populationSize,
                    const double epsilon,
                    const arma::vec& lowerBound,
                    const arma::vec& upperBound) :
+    numObjectives(0),
+    numVariables(0),
    populationSize(populationSize),
    maxGenerations(maxGenerations),
    crossoverProb(crossoverProb),
@@ -45,6 +48,8 @@ inline NSGA2::NSGA2(const size_t populationSize,
                    const double epsilon,
                    const double lowerBound,
                    const double upperBound) :
+    numObjectives(0),
+    numVariables(0),
    populationSize(populationSize),
    maxGenerations(maxGenerations),
    crossoverProb(crossoverProb),
@@ -61,7 +66,7 @@ template<typename MatType,
         typename... CallbackTypes>
 typename MatType::elem_type NSGA2::Optimize(
    std::tuple<ArbitraryFunctionType...>& objectives,
-    MatType& iterate,
+    MatType& iterateIn,
    CallbackTypes&&... callbacks)
 {
  // Make sure for evolution to work at least four candidates are present.
@@ -71,6 +76,17 @@ typename MatType::elem_type NSGA2::Optimize(
        " least 4, and, a multiple of 4!");
  }

+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  // Make sure that we have the methods that we need.  Long name...
+  traits::CheckArbitraryFunctionTypeAPI<ArbitraryFunctionType...,
+      BaseMatType>();
+  RequireDenseFloatingPointType<BaseMatType>();
+
  // Check if lower bound is a vector of a single dimension.
  if (lowerBound.n_rows == 1)
    lowerBound = lowerBound(0, 0) * arma::ones(iterate.n_rows, iterate.n_cols);
@@ -85,29 +101,29 @@ typename MatType::elem_type NSGA2::Optimize(
  assert(upperBound.n_rows == iterate.n_rows && "The dimensions of "
      "upperBound are not the same as the dimensions of iterate.");

-  // Convenience typedefs.
-  typedef typename MatType::elem_type ElemType;
-
  numObjectives = sizeof...(ArbitraryFunctionType);
  numVariables = iterate.n_rows;

  // Cache calculated objectives.
-  std::vector<arma::Col<ElemType> > calculatedObjectives;
-  // Pre-allocate space for the calculated objectives.
-  calculatedObjectives.resize(populationSize);
+  std::vector<arma::Col<ElemType> > calculatedObjectives(populationSize);

  // Population size reserved to 2 * populationSize + 1 to accommodate
  // for the size of intermediate candidate population.
-  std::vector<MatType> population;
+  std::vector<BaseMatType> population;
  population.reserve(2 * populationSize + 1);

  // Pareto fronts, initialized during non-dominated sorting.
+  // Stores indices of population belonging to a certain front.
  std::vector<std::vector<size_t> > fronts;
  // Initialised in CrowdingDistanceAssignment.
-  std::vector<double> crowdingDistance;
+  std::vector<ElemType> crowdingDistance;
  // Initialised during non-dominated sorting.
  std::vector<size_t> ranks;

+  //! Useful temporaries for float-like comparisons.
+  const BaseMatType castedLowerBound = arma::conv_to<BaseMatType>::from(lowerBound);
+  const BaseMatType castedUpperBound = arma::conv_to<BaseMatType>::from(upperBound);
+
  // Controls early termination of the optimization process.
  bool terminate = false;

@@ -115,87 +131,103 @@ typename MatType::elem_type NSGA2::Optimize(
  // starting point.
  for (size_t i = 0; i < populationSize; i++)
  {
-    population.push_back(arma::randu<MatType>(iterate.n_rows,
+    population.push_back(arma::randu<BaseMatType>(iterate.n_rows,
        iterate.n_cols) - 0.5 + iterate);
+
+    // Constrain all genes to be within bounds.
+    population[i] = arma::min(arma::max(population[i], castedLowerBound), castedUpperBound);
  }

  Info << "NSGA2 initialized successfully. Optimization started." << std::endl;

-  // Evaluate the fitness before optimization.
-  for (size_t i = 0; i < population.size(); i++)
-    calculatedObjectives[i] = arma::Col<ElemType>(numObjectives, arma::fill::zeros);
-  EvaluateObjectives(population, objectives, calculatedObjectives);
-
  // Iterate until maximum number of generations is obtained.
-  terminate |= Callback::BeginOptimization(*this, objectives, iterate, callbacks...);
+  Callback::BeginOptimization(*this, objectives, iterate, callbacks...);

  for (size_t generation = 1; generation <= maxGenerations && !terminate; generation++)
  {
    Info << "NSGA2: iteration " << generation << "." << std::endl;
-    terminate |= Callback::StepTaken(*this, objectives, iterate, callbacks...);

    // Create new population of candidate from the present elite population.
    // Have P_t, generate G_t using P_t.
-    BinaryTournamentSelection(population, lowerBound, upperBound);
+    BinaryTournamentSelection(population, castedLowerBound, castedUpperBound);

    // Evaluate the objectives for the new population.
    calculatedObjectives.resize(population.size());
-    for (size_t i = 0; i < population.size(); i++)
-      calculatedObjectives[i] = arma::Col<ElemType>(numObjectives, arma::fill::zeros);
+    std::fill(calculatedObjectives.begin(), calculatedObjectives.end(),
+        arma::Col<ElemType>(numObjectives, arma::fill::zeros));
    EvaluateObjectives(population, objectives, calculatedObjectives);

    // Perform fast non dominated sort on P_t ∪ G_t.
    ranks.resize(population.size());
-    FastNonDominatedSort<MatType>(fronts, ranks, calculatedObjectives);
+    FastNonDominatedSort<BaseMatType>(fronts, ranks, calculatedObjectives);

    // Perform crowding distance assignment.
    crowdingDistance.resize(population.size());
-
+    std::fill(crowdingDistance.begin(), crowdingDistance.end(), 0.);
    for (size_t fNum = 0; fNum < fronts.size(); fNum++)
    {
-      CrowdingDistanceAssignment(fronts[fNum], crowdingDistance);
+      CrowdingDistanceAssignment<BaseMatType>(
+          fronts[fNum], calculatedObjectives, crowdingDistance);
    }

    // Sort based on crowding distance.
    std::sort(population.begin(), population.end(),
-              [this, ranks, crowdingDistance, population](MatType candidateP,
-                                                          MatType candidateQ)
-              {
-                size_t idxP, idxQ;
-                for (size_t i = 0; i < population.size(); i++)
-                {
-                  if (arma::approx_equal(population[i], candidateP, "absdiff", epsilon))
-                    idxP = i;
+      [this, ranks, crowdingDistance, population]
+        (BaseMatType candidateP, BaseMatType candidateQ)
+          {
+            size_t idxP{}, idxQ{};
+            for (size_t i = 0; i < population.size(); i++)
+            {
+              if (arma::approx_equal(population[i], candidateP, "absdiff", epsilon))
+                idxP = i;

-                  if (arma::approx_equal(population[i], candidateQ, "absdiff", epsilon))
-                    idxQ = i;
-                }
+              if (arma::approx_equal(population[i], candidateQ, "absdiff", epsilon))
+                idxQ = i;
+            }

-                return CrowdingOperator(idxP, idxQ, ranks, crowdingDistance);
-              }
+            return CrowdingOperator<BaseMatType>(idxP, idxQ, ranks, crowdingDistance);
+          }
    );

    // Yield a new population P_{t+1} of size populationSize.
+    // Discards unfit population from the R_{t} to yield P_{t+1}.
    population.resize(populationSize);
+
+    terminate |= Callback::GenerationalStepTaken(*this, objectives, iterate,
+        calculatedObjectives, fronts, callbacks...);
  }

-  // Set the candidates from the best front as the output.
-  std::vector<MatType> front;
+  // Set the candidates from the Pareto Set as the output.
+  paretoSet.set_size(population[0].n_rows, population[0].n_cols, fronts[0].size());
+  // The Pareto Set is stored, can be obtained via ParetoSet() getter.
+  for (size_t solutionIdx = 0; solutionIdx < fronts[0].size(); ++solutionIdx)
+  {
+    paretoSet.slice(solutionIdx) =
+      arma::conv_to<arma::mat>::from(population[fronts[0][solutionIdx]]);
+  }

-  for (size_t f: fronts[0])
-    front.push_back(population[f]);
+  // Set the candidates from the Pareto Front as the output.
+  paretoFront.set_size(calculatedObjectives[0].n_rows, calculatedObjectives[0].n_cols,
+      fronts[0].size());
+  // The Pareto Front is stored, can be obtained via ParetoFront() getter.
+  for (size_t solutionIdx = 0; solutionIdx < fronts[0].size(); ++solutionIdx)
+  {
+    paretoFront.slice(solutionIdx) =
+      arma::conv_to<arma::mat>::from(calculatedObjectives[fronts[0][solutionIdx]]);
+  }

-  // bestFront is stored, can be obtained by the Front() getter.
-  bestFront = front;
+  // Clear rcFront, in case it is later requested by the user for reverse
+  // compatibility reasons.
+  rcFront.clear();

-  // Assign iterate to first element of the best front.
-  iterate = bestFront[0];
+  // Assign iterate to first element of the Pareto Set.
+  iterate = population[fronts[0][0]];

  Callback::EndOptimization(*this, objectives, iterate, callbacks...);

  ElemType performance = std::numeric_limits<ElemType>::max();

-  for(arma::Col<ElemType> objective: calculatedObjectives)
+  for (const arma::Col<ElemType>& objective: calculatedObjectives)
    if (arma::accu(objective) < performance)
      performance = arma::accu(objective);

@@ -210,7 +242,7 @@ typename std::enable_if<I == sizeof...(ArbitraryFunctionType), void>::type
 NSGA2::EvaluateObjectives(
    std::vector<MatType>&,
    std::tuple<ArbitraryFunctionType...>&,
-    std::vector<arma::Col<double> >&)
+    std::vector<arma::Col<typename MatType::elem_type> >&)
 {
  // Nothing to do here.
 }
@@ -223,7 +255,7 @@ typename std::enable_if<I < sizeof...(ArbitraryFunctionType), void>::type
 NSGA2::EvaluateObjectives(
    std::vector<MatType>& population,
    std::tuple<ArbitraryFunctionType...>& objectives,
-    std::vector<arma::Col<double> >& calculatedObjectives)
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives)
 {
  for (size_t i = 0; i < populationSize; i++)
  {
@@ -236,8 +268,8 @@ NSGA2::EvaluateObjectives(
 //! Reproduce and generate new candidates.
 template<typename MatType>
 inline void NSGA2::BinaryTournamentSelection(std::vector<MatType>& population,
-                                             const arma::vec& lowerBound,
-                                             const arma::vec& upperBound)
+                                             const MatType& lowerBound,
+                                             const MatType& upperBound)
 {
  std::vector<MatType> children;

@@ -292,20 +324,14 @@ inline void NSGA2::Crossover(MatType& childA,
 //! Perform mutation of the candidates weights with some noise.
 template<typename MatType>
 inline void NSGA2::Mutate(MatType& child,
-                          const arma::vec& lowerBound,
-                          const arma::vec& upperBound)
+                          const MatType& lowerBound,
+                          const MatType& upperBound)
 {
  child += (arma::randu<MatType>(child.n_rows, child.n_cols) < mutationProb) %
      (mutationStrength * arma::randn<MatType>(child.n_rows, child.n_cols));

  // Constrain all genes to be between bounds.
-  for (size_t idx = 0; idx < numVariables; idx++)
-  {
-    if (child[idx] < lowerBound(idx))
-      child[idx] = lowerBound(idx);
-    else if (child[idx] > upperBound(idx))
-      child[idx] = upperBound(idx);
-  }
+  child = arma::min(arma::max(child, lowerBound), upperBound);
 }

 //! Sort population into Pareto fronts.
@@ -344,7 +370,7 @@ inline void NSGA2::FastNonDominatedSort(

  size_t i = 0;

-  while (fronts[i].size() > 0)
+  while (!fronts[i].empty())
  {
    std::vector<size_t> nextFront;

@@ -365,6 +391,8 @@ inline void NSGA2::FastNonDominatedSort(
    i++;
    fronts.push_back(nextFront);
  }
+  // Remove the empty final set.
+  fronts.pop_back();
 }

 //! Check if a candidate Pareto dominates another candidate.
@@ -393,37 +421,59 @@ inline bool NSGA2::Dominates(
 }

 //! Assign crowding distance to the population.
-inline void NSGA2::CrowdingDistanceAssignment(const std::vector<size_t>& front,
-                                              std::vector<double>& crowdingDistance)
+template <typename MatType>
+inline void NSGA2::CrowdingDistanceAssignment(
+    const std::vector<size_t>& front,
+    std::vector<arma::Col<typename MatType::elem_type>>& calculatedObjectives,
+    std::vector<typename MatType::elem_type>& crowdingDistance)
 {
-  if (front.size() > 0)
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+
+  size_t fSize = front.size();
+  // Stores the sorted indices of the fronts.
+  arma::uvec sortedIdx  = arma::regspace<arma::uvec>(0, 1, fSize - 1);
+
+  for (size_t m = 0; m < numObjectives; m++)
  {
-    for (size_t elem: front)
-      crowdingDistance[elem] = 0;
+    // Cache fValues of individuals for current objective.
+    arma::Col<ElemType> fValues(fSize);
+    std::transform(front.begin(), front.end(), fValues.begin(),
+      [&](const size_t& individual)
+        {
+          return calculatedObjectives[individual](m);
+        });

-    size_t fSize = front.size();
+    // Sort front indices by ascending fValues for current objective.
+    std::sort(sortedIdx.begin(), sortedIdx.end(),
+      [&](const size_t& frontIdxA, const size_t& frontIdxB)
+        {
+          return (fValues(frontIdxA) < fValues(frontIdxB));
+        });

-    for (size_t m = 0; m < numObjectives; m++)
+    crowdingDistance[front[sortedIdx(0)]] =
+        std::numeric_limits<ElemType>::max();
+    crowdingDistance[front[sortedIdx(fSize - 1)]] =
+        std::numeric_limits<ElemType>::max();
+    ElemType minFval = fValues(sortedIdx(0));
+    ElemType maxFval = fValues(sortedIdx(fSize - 1));
+    ElemType scale =
+        std::abs(maxFval - minFval) == 0. ? 1. : std::abs(maxFval - minFval);
+
+    for (size_t i = 1; i < fSize - 1; i++)
    {
-      crowdingDistance[front[0]] = std::numeric_limits<double>::max();
-      crowdingDistance[front[fSize - 1]] = std::numeric_limits<double>::max();
-
-      for (size_t i = 1; i < fSize - 1 ; i++)
-      {
-        crowdingDistance[front[i]] += (crowdingDistance[front[i - 1]] -
-            crowdingDistance[front[i + 1]]) /
-            (std::numeric_limits<double>::max() -
-            std::numeric_limits<double>::min());
-      }
+      crowdingDistance[front[sortedIdx(i)]] +=
+          (fValues(sortedIdx(i + 1)) - fValues(sortedIdx(i - 1))) / scale;
    }
  }
 }

 //! Comparator for crowding distance based sorting.
+template<typename MatType>
 inline bool NSGA2::CrowdingOperator(size_t idxP,
                                    size_t idxQ,
                                    const std::vector<size_t>& ranks,
-                                    const std::vector<double>& crowdingDistance)
+                                    const std::vector<typename MatType::elem_type>& crowdingDistance)
 {
  if (ranks[idxP] < ranks[idxQ])
    return true;
@@ -50,8 +50,7 @@ class PadamUpdate
      epsilon(epsilon),
      beta1(beta1),
      beta2(beta2),
-      partial(partial),
-      iteration(0)
+      partial(partial)
  {
    // Nothing to do.
  }
@@ -76,11 +75,6 @@ class PadamUpdate
  //! Modify the partial adaptive parameter.
  double& Partial() { return partial; }

-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
  /**
   * The UpdatePolicyType policy classes must contain an internal 'Policy'
   * template class with two template arguments: MatType and GradType.  This is
@@ -100,7 +94,8 @@ class PadamUpdate
     * @param cols Number of columns in the gradient matrix.
     */
    Policy(PadamUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        iteration(0)
    {
      m.zeros(rows, cols);
      v.zeros(rows, cols);
@@ -119,7 +114,7 @@ class PadamUpdate
                const GradType& gradient)
    {
      // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;

      // And update the iterate.
      m *= parent.beta1;
@@ -128,10 +123,8 @@ class PadamUpdate
      v *= parent.beta2;
      v += (1 - parent.beta2) * (gradient % gradient);

-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);

      // Element wise maximum of past and present squared gradients.
      vImproved = arma::max(vImproved, v);
@@ -152,6 +145,9 @@ class PadamUpdate

    //! The optimal sqaured gradient value.
    GradType vImproved;
+
+    //! The number of iterations.
+    size_t iteration;
  };

 private:
@@ -166,9 +162,6 @@ class PadamUpdate

  //! Partial adaptive parameter.
  double partial;
-
-  //! The number of iterations.
-  size_t iteration;
 };

 } // namespace ens
@@ -98,8 +98,7 @@ typename MatType::elem_type>::type ParallelSGD<DecayPolicyType>::Optimize(
  // Iterate till the objective is within tolerance or the maximum number of
  // allowed iterations is reached. If maxIterations is 0, this will iterate
  // till convergence.
-  terminate |= Callback::BeginOptimization(*this, function, iterate,
-      callbacks...);
+  Callback::BeginOptimization(*this, function, iterate, callbacks...);
  for (size_t i = 1; i != maxIterations && !terminate; ++i)
  {
    // Calculate the overall objective.
@@ -0,0 +1,216 @@
+/**
+ * @file dtlz1_function.hpp
+ * @author Satyam Shukla
+ *
+ * Implementation of the first DTLZ(Deb, Thiele, Laumanns, and Zitzler) test.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_DTLZ_ONE_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_DTLZ_ONE_FUNCTION_HPP
+
+#include "../../moead/weight_init_policies/uniform_init.hpp"
+
+namespace ens {
+namespace test {
+
+/**
+ * The DTLZ1 function, defined by:
+ * \f[
+ * x_M = [x_i, n - M + 1 <= i <= n]
+ * g(x) = 100 * [|x_M| + \Sigma{i = n - M + 1}^n (x_i - 0.5)^2 - cos(20 * pi * 
+ *   (x_i - 0.5))]
+ * 
+ * f_1(x) = 0.5 * x_1 * x_2 * ... x_M-1 * (1 + g(x_M)) 
+ * f_2(x) = 0.5 * x_1 * x_2 * ... (1 - x_M-1) * (1 + g(x_M))
+ * .
+ * .
+ * f_M(x) = 0.5 * (1 - x_1) * (1 + g(x_M))
+ * \f]
+ *
+ * Bounds of the variable space is:
+ * 0 <= x_i <= 1 for i = 1,...,n.
+ *
+ * This should be optimized to x_i = 0.5 (for all x_i in x_M), at:
+ * the objective function values lie on the linear hyper-plane: 
+ * \Sigma { m = 1}^M f_m* =0.5.
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @incollection{deb2005scalable,
+ * title={Scalable test problems for evolutionary multiobjective optimization},
+ * author={Deb, Kalyanmoy and Thiele, Lothar and Laumanns, Marco and Zitzler, Eckart},
+ * booktitle={Evolutionary multiobjective optimization: theoretical advances and applications},
+ * pages={105--145},
+ * year={2005},
+ * publisher={Springer}
+ * }
+ * @endcode
+ *
+ * @tparam MatType Type of matrix to optimize.
+ */
+  template <typename MatType = arma::mat>
+  class DTLZ1
+  {
+    private:
+
+    // A fixed no. of Objectives and Variables(|x| = 7, M = 3).
+    size_t numObjectives {3};
+    size_t numVariables {7};
+    size_t numParetoPoints;
+
+    public:
+
+      /**
+       * Object Constructor.
+       * Initializes the individual objective functions.
+       *
+       * @param numParetoPoint No. of pareto points in the reference front.
+       */
+      DTLZ1 (size_t numParetoPoint = 136) :
+        numParetoPoints(numParetoPoint),
+        objectiveF1(0, *this),
+        objectiveF2(1, *this),
+        objectiveF3(2, *this)
+      {/* Nothing to do here */}
+
+      // Get the private variables.
+      
+      // Get the number of objectives.
+      size_t GetNumObjectives()
+      { return this -> numObjectives; }
+
+      // Get the number of Variables.
+      size_t GetNumVariables()
+      { return this -> numVariables; }
+
+      /**
+      * Set the no. of pareto points.
+      *
+      * @param numParetoPoint No. of pareto points in the reference front.
+      */
+      void SetNumParetoPoint (size_t numParetoPoint)
+      { this -> numParetoPoints = numParetoPoint; }
+
+      // Get the starting point.
+      arma::Col<typename MatType::elem_type> GetInitialPoint()
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+        return arma::Col<ElemType>(numVariables, arma::fill::ones);
+      }
+
+      /**
+      * Evaluate the G(x) with the given coordinate.
+      *
+      * @param coords The function coordinates.
+      * @return arma::Row<typename MatType::elem_type>
+      */
+      arma::Row<typename MatType::elem_type> g(const MatType& coords)
+      {
+        size_t k = numVariables - numObjectives + 1;
+
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+        
+        arma::Row<ElemType> innerSum(size(coords)[1], arma::fill::zeros);
+        
+        for (size_t i = numObjectives - 1; i < numVariables; i++)
+        {
+          innerSum += arma::pow((coords.row(i) - 0.5), 2) - 
+              arma::cos(20 * arma::datum::pi * (coords.row(i) - 0.5)); 
+        } 
+        
+        return 100 * (k + innerSum);
+      }     
+
+      /**
+      * Evaluate the objectives with the given coordinate.
+      *
+      * @param coords The function coordinates.
+      * @return arma::Mat<typename MatType::elem_type>
+      */
+      arma::Mat<typename MatType::elem_type> Evaluate(const MatType& coords)
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+
+        arma::Mat<ElemType> objectives(numObjectives, size(coords)[1]);
+        arma::Row<ElemType> G = g(coords);
+        arma::Row<ElemType> value = 0.5 * (1. + G);
+        for (size_t i = 0; i < numObjectives - 1; i++)
+        {
+          objectives.row(i) =  value % (1.0 - coords.row(i));
+          value = value % coords.row(i);
+        }
+        objectives.row(numObjectives - 1) = value;
+        return objectives;    
+      }
+      
+      // Individual Objective function.
+      // Changes based on stop variable provided. 
+      struct DTLZObjective
+      {
+        DTLZObjective(size_t stop, DTLZ1<MatType>& dtlz) : stop(stop), dtlz(dtlz)
+        {/* Nothing to do here. */}  
+        
+        /**
+        * Evaluate one objective with the given coordinate.
+        *
+        * @param coords The function coordinates.
+        * @return arma::Col<typename MatType::elem_type>
+        */
+        typename MatType::elem_type Evaluate(const MatType& coords)
+        {
+          // Convenience typedef.
+          typedef typename MatType::elem_type ElemType;
+          ElemType value = 0.5;
+          for (size_t i = 0; i < stop; i++)
+          {
+            value = value * coords[i];
+          }
+
+          if(stop != dtlz.numObjectives - 1)
+          {
+            value = value * (1. - coords[stop]);
+          }
+          else
+          {
+            value = value * coords[stop];
+          }
+
+          value = value * (1. + dtlz.g(coords)[0]);
+          return value; 
+        }        
+
+        DTLZ1& dtlz;
+        size_t stop;
+      };
+
+      // Return back a tuple of objective functions.
+      std::tuple<DTLZObjective, DTLZObjective, DTLZObjective> GetObjectives()
+      {
+          return std::make_tuple(objectiveF1, objectiveF2, objectiveF3);
+      } 
+
+      //! Get the Reference Front.
+      //! Front. The implementation has been taken from pymoo.
+      arma::mat GetReferenceFront()
+      {
+        Uniform refGenerator;
+        return 0.5 * refGenerator.Generate<arma::mat>(3, numParetoPoints, 0);
+      }
+
+    DTLZObjective objectiveF1;
+    DTLZObjective objectiveF2;
+    DTLZObjective objectiveF3;
+  };
+  } //namespace test
+  } //namespace ens
+
+#endif
@@ -0,0 +1,217 @@
+/**
+ * @file dtlz2_function.hpp
+ * @author Satyam Shukla
+ *
+ * Implementation of the second DTLZ(Deb, Thiele, Laumanns, and Zitzler) test.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_DTLZ_TWO_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_DTLZ_TWO_FUNCTION_HPP
+
+#include "../../moead/weight_init_policies/uniform_init.hpp"
+
+namespace ens {
+namespace test {
+
+/**
+ * The DTLZ2 function, defined by:
+ * \f[
+ * x_M = [x_i, n - M + 1 <= i <= n]
+ * g(x) = \Sigma{i = n - M + 1}^n (x_i - 0.5)^2
+ * 
+ * f_1(x) = 0.5 * cos(x_1 * pi * 0.5) * cos(x_2 * pi * 0.5) * ... cos(x_2 * pi * 0.5) * (1 + g(x_M)) 
+ * f_2(x) = 0.5 * cos(x_1 * pi * 0.5) * cos(x_2 * pi * 0.5) * ... sin(x_M-1 * pi * 0.5) * (1 + g(x_M))
+ * .
+ * .
+ * f_M(x) = 0.5 * sin(x_1 * pi * 0.5) * (1 + g(x_M))
+ * \f]
+ *
+ * Bounds of the variable space is:
+ * 0 <= x_i <= 1 for i = 1,...,n.
+ *
+ * This should be optimized to x_i = 0.5 (for all x_i in x_M), at:
+ * 
+ * For more information, please refer to:
+ * 
+ * @code
+ * @incollection{deb2005scalable,
+ * title={Scalable test problems for evolutionary multiobjective optimization},
+ * author={Deb, Kalyanmoy and Thiele, Lothar and Laumanns, Marco and Zitzler, Eckart},
+ * booktitle={Evolutionary multiobjective optimization: theoretical advances and applications},
+ * pages={105--145},
+ * year={2005},
+ * publisher={Springer}
+ * }
+ * @endcode
+ *
+ * @tparam MatType Type of matrix to optimize.
+ */
+  template <typename MatType = arma::mat>
+  class DTLZ2
+  {
+    private:
+
+    // A fixed no. of Objectives and Variables(|x| = 7, M = 3).
+    size_t numObjectives {3};
+    size_t numVariables {7};
+    size_t numParetoPoints;
+
+    public:
+      
+      /**
+       * Object Constructor.
+       * Initializes the individual objective functions.
+       *
+       * @param numParetoPoint No. of pareto points in the reference front.
+       */
+      DTLZ2(size_t numParetoPoints = 136) :
+        numParetoPoints(numParetoPoints),
+        objectiveF1(0, *this),
+        objectiveF2(1, *this),
+        objectiveF3(2, *this)
+      {/*Nothing to do here.*/}
+
+      //! Get the starting point.
+      arma::Col<typename MatType::elem_type> GetInitialPoint()
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+        return arma::Col<ElemType>(numVariables, 1, arma::fill::zeros);
+      } 
+      
+      // Get the private variables.
+      
+      // Get the number of objectives.
+      size_t GetNumObjectives ()
+      { return this -> numObjectives; }
+
+      // Get the number of variables.
+      size_t GetNumVariables ()
+      { return this -> numVariables; }
+
+      /**
+       * Set the no. of pareto points.
+       *
+       * @param numParetoPoint
+       */
+      void SetNumParetoPoint (size_t numParetoPoint)
+      { this -> numParetoPoints = numParetoPoint; }
+
+      /**
+       * Evaluate the G(x) with the given coordinate.
+       *
+       * @param coords The function coordinates.
+       * @return arma::Row<typename MatType::elem_type>
+       */
+      arma::Row<typename MatType::elem_type> g(const MatType& coords)
+      {
+        size_t k = numVariables - numObjectives + 1;
+
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+        
+        arma::Row<ElemType> innerSum(size(coords)[1], arma::fill::zeros);
+        
+        for(size_t i = numObjectives - 1; i < numVariables; i++)
+        {
+          innerSum += arma::pow((coords.row(i) - 0.5), 2); 
+        } 
+        
+        return innerSum;
+      }    
+
+      /**
+       * Evaluate the objectives with the given coordinate.
+       *
+       * @param coords The function coordinates.
+       * @return arma::Mat<typename MatType::elem_type>
+       */
+      arma::Mat<typename MatType::elem_type> Evaluate(const MatType& coords)
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+
+        arma::Mat<ElemType> objectives(numObjectives, size(coords)[1]);
+        arma::Row<ElemType> G = g(coords);
+        arma::Row<ElemType> value = 0.5 * (1.0 + G);
+        for(size_t i = 0; i < numObjectives - 1; i++)
+        {
+          objectives.row(i) =  value %  
+              arma::sin(coords.row(i) * arma::datum::pi * 0.5);
+          value = value % arma::cos(coords.row(i) * arma::datum::pi * 0.5); 
+        }
+        objectives.row(numObjectives - 1) = value;
+        return objectives;    
+      }
+      
+      // Individual Objective function.
+      // Changes based on stop variable provided. 
+      struct DTLZ2Objective
+      {
+        DTLZ2Objective(size_t stop, DTLZ2& dtlz): stop(stop), dtlz(dtlz)
+        {/* Nothing to do here. */}  
+        
+        /**
+         * Evaluate one objective with the given coordinate.
+         *
+         * @param coords The function coordinates.
+         * @return arma::Col<typename MatType::elem_type>
+         */
+        typename MatType::elem_type Evaluate(const MatType& coords)
+        {
+          // Convenience typedef.
+          typedef typename MatType::elem_type ElemType;
+          ElemType value = 1.0;
+          for(size_t i = 0; i < stop; i++)
+          {
+            value = value * std::cos(coords[i] * arma::datum::pi * 0.5);
+          }
+
+          if(stop != dtlz.numObjectives - 1)
+          {
+            value = value * std::sin(coords[stop] * arma::datum::pi * 0.5);
+          }
+          else
+          {
+            value = value * std::cos(coords[stop] * arma::datum::pi * 0.5);
+          }
+
+          value = value * (1.0 + dtlz.g(coords)[0]);
+          return value;  
+        }        
+
+        DTLZ2& dtlz;
+        size_t stop;
+      };
+
+      // Return back a tuple of objective functions.
+      std::tuple<DTLZ2Objective, DTLZ2Objective, DTLZ2Objective> GetObjectives()
+      {
+          return std::make_tuple(objectiveF1, objectiveF2, objectiveF3);
+      } 
+
+      //! Get the Reference Front.
+      //! Front. The implementation has been taken from pymoo.
+      arma::mat GetReferenceFront()
+      { 
+      	Uniform refGenerator;
+        arma::mat refDirs = refGenerator.Generate<arma::mat>(3, this -> numParetoPoints, 0);
+        arma::colvec x = arma::normalise(refDirs, 2, 1);
+        arma::mat A(size(refDirs), arma::fill::ones);
+        A.each_col() = x;
+        return refDirs / A;
+      }
+
+    DTLZ2Objective objectiveF1;
+    DTLZ2Objective objectiveF2;
+    DTLZ2Objective objectiveF3;
+  };
+  } //namespace test
+  } //namespace ens
+
+#endif
@@ -0,0 +1,219 @@
+/**
+ * @file dtlz3_function.hpp
+ * @author Satyam Shukla
+ *
+ * Implementation of the third DTLZ(Deb, Thiele, Laumanns, and Zitzler) test.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_DTLZ_THREE_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_DTLZ_THREE_FUNCTION_HPP
+
+#include "../../moead/weight_init_policies/uniform_init.hpp"
+
+namespace ens {
+namespace test {
+
+/**
+ * The DTLZ3 function, defined by:
+ * \f[
+ * x_M = [x_i, n - M + 1 <= i <= n]
+ * g(x) = 100 * [|x_M| + \Sigma{i = n - M + 1}^n (x_i - 0.5)^2 - cos(20 * pi * 
+ *   (x_i - 0.5))]
+ * 
+ * f_1(x) = 0.5 * cos(x_1 * pi * 0.5) * cos(x_2 * pi * 0.5) * ... cos(x_2 * pi * 0.5) * (1 + g(x_M)) 
+ * f_2(x) = 0.5 * cos(x_1 * pi * 0.5) * cos(x_2 * pi * 0.5) * ... sin(x_M-1 * pi * 0.5) * (1 + g(x_M))
+ * .
+ * .
+ * f_M(x) = 0.5 * sin(x_1 * pi * 0.5) * (1 + g(x_M))
+ * \f]
+ *
+ * Bounds of the variable space is:
+ * 0 <= x_i <= 1 for i = 1,...,n.
+ *
+ * This should be optimized to x_i = 0.5 (for all x_i in x_M), at:
+ * 
+ * For more information, please refer to:
+ * 
+ * @code
+ * @incollection{deb2005scalable,
+ * title={Scalable test problems for evolutionary multiobjective optimization},
+ * author={Deb, Kalyanmoy and Thiele, Lothar and Laumanns, Marco and Zitzler, Eckart},
+ * booktitle={Evolutionary multiobjective optimization: theoretical advances and applications},
+ * pages={105--145},
+ * year={2005},
+ * publisher={Springer}
+ * }
+ * @endcode
+ *
+ * @tparam MatType Type of matrix to optimize.
+ */
+  template <typename MatType = arma::mat>
+  class DTLZ3
+  {
+    private:
+
+    // A fixed no. of Objectives and Variables(|x| = 7, M = 3).
+    size_t numObjectives {3};
+    size_t numVariables {7};
+    size_t numParetoPoints;
+
+    public:
+
+      /**
+       * Object Constructor.
+       * Initializes the individual objective functions.
+       *
+       * @param numParetoPoint No. of pareto points in the reference front.
+       */
+      DTLZ3(size_t numParetoPoints = 136) :
+        numParetoPoints(numParetoPoints),
+        objectiveF1(0, *this),
+        objectiveF2(1, *this),
+        objectiveF3(2, *this)
+      {/*Nothing to do here.*/}
+
+      //! Get the starting point.
+      arma::Col<typename MatType::elem_type> GetInitialPoint()
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+        return arma::Col<ElemType>(numVariables, 1, arma::fill::zeros);
+      } 
+      
+      // Get the private variables.
+      
+      // Get the number of objectives.
+      size_t GetNumObjectives()
+      { return this -> numObjectives; }
+
+      // Get the number of variables.
+      size_t GetNumVariables()
+      { return this -> numVariables;}
+
+      /**
+       * Set the no. of pareto points.
+       *
+       * @param numParetoPoint No. of pareto points in the reference front.
+       */
+      void SetNumParetoPoint(size_t numParetoPoint)
+      { this -> numParetoPoints = numParetoPoint;}
+
+      /**
+       * Evaluate the G(x) with the given coordinate.
+       *
+       * @param coords The function coordinates.
+       * @return arma::Row<typename MatType::elem_type>
+       */
+      arma::Row<typename MatType::elem_type> g(const MatType& coords)
+      {
+        size_t k = numVariables - numObjectives + 1;
+
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+        
+        arma::Row<ElemType> innerSum(size(coords)[1], arma::fill::zeros);
+        
+        for(size_t i = numObjectives - 1; i < numVariables; i++)
+        {
+          innerSum += arma::pow((coords.row(i) - 0.5), 2) - 
+              arma::cos(20 * arma::datum::pi * (coords.row(i) - 0.5)); 
+        } 
+        
+        return 100 * (k + innerSum);
+      }    
+
+      /**
+       * Evaluate the objectives with the given coordinate.
+       *
+       * @param coords The function coordinates.
+       * @return arma::Mat<typename MatType::elem_type>
+       */
+      arma::Mat<typename MatType::elem_type> Evaluate(const MatType& coords)
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+
+        arma::Mat<ElemType> objectives(numObjectives, size(coords)[1]);
+        arma::Row<ElemType> G = g(coords);
+        arma::Row<ElemType> value = 0.5 * (1.0 + G);
+        for(size_t i = 0; i < numObjectives - 1; i++)
+        {
+          objectives.row(i) =  value %  
+              arma::sin(coords.row(i) * arma::datum::pi * 0.5);
+          value = value % arma::cos(coords.row(i) * arma::datum::pi * 0.5); 
+        }
+        objectives.row(numObjectives - 1) = value;
+        return objectives;    
+      }
+      
+      // Individual Objective function.
+      // Changes based on stop variable provided. 
+      struct DTLZ3Objective
+      {
+        DTLZ3Objective(size_t stop, DTLZ3& dtlz): stop(stop), dtlz(dtlz)
+        {/* Nothing to do here. */}  
+        
+        /**
+         * Evaluate one objective with the given coordinate.
+         *
+         * @param coords The function coordinates.
+         * @return arma::Col<typename MatType::elem_type>
+         */
+        typename MatType::elem_type Evaluate(const MatType& coords)
+        {
+          // Convenience typedef.
+          typedef typename MatType::elem_type ElemType;
+          ElemType value = 1.0;
+          for(size_t i = 0; i < stop; i++)
+          {
+            value = value * std::cos(coords[i] * arma::datum::pi * 0.5);
+          }
+
+          if(stop != dtlz.numObjectives - 1)
+          {
+            value = value * std::sin(coords[stop] * arma::datum::pi * 0.5);
+          }
+          else
+          {
+            value = value * std::cos(coords[stop] * arma::datum::pi * 0.5);
+          }
+
+          value = value * (1. + dtlz.g(coords)[0]);
+          return value;  
+        }        
+
+        DTLZ3& dtlz;
+        size_t stop;
+      };
+
+      // Return back a tuple of objective functions.
+      std::tuple<DTLZ3Objective, DTLZ3Objective, DTLZ3Objective> GetObjectives()
+      {
+          return std::make_tuple(objectiveF1, objectiveF2, objectiveF3);
+      } 
+
+      //! Get the Reference Front.
+      //! Front. The implementation has been taken from pymoo.
+      arma::mat GetReferenceFront()
+      { 
+      	Uniform refGenerator;
+        arma::mat refDirs = refGenerator.Generate<arma::mat>(3, this -> numParetoPoints, 0);
+        arma::colvec x = arma::normalise(refDirs, 2, 1);
+        arma::mat A(size(refDirs), arma::fill::ones);
+        A.each_col() = x;
+        return refDirs / A;
+      }
+
+    DTLZ3Objective objectiveF1;
+    DTLZ3Objective objectiveF2;
+    DTLZ3Objective objectiveF3;
+  };
+  } //namespace test
+  } //namespace ens
+
+#endif
@@ -0,0 +1,220 @@
+/**
+ * @file dtlz4_function.hpp
+ * @author Satyam Shukla
+ *
+ * Implementation of the fourth DTLZ(Deb, Thiele, Laumanns, and Zitzler) test.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_DTLZ_FOUR_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_DTLZ_FOUR_FUNCTION_HPP
+
+#include "../../moead/weight_init_policies/uniform_init.hpp"
+
+namespace ens {
+namespace test {
+
+/**
+ * The DTLZ4 function, defined by:
+ * \f[
+ * x_M = [x_i, n - M + 1 <= i <= n]
+ * g(x) = \Sigma{i = n - M + 1}^n (x_i - 0.5)^2
+ * 
+ * f_1(x) = 0.5 * cos(x_1^alpha * pi * 0.5) * cos(x_2^alpha * pi * 0.5) * ... cos(x_2^alpha * pi * 0.5) * (1 + g(x_M)) 
+ * f_2(x) = 0.5 * cos(x_1^alpha * pi * 0.5) * cos(x_2^alpha * pi * 0.5) * ... sin(x_M-1^alpha * pi * 0.5) * (1 + g(x_M))
+ * .
+ * .
+ * f_M(x) = 0.5 * sin(x_1^alpha * pi * 0.5) * (1 + g(x_M))
+ * \f]
+ *
+ * Bounds of the variable space is:
+ * 0 <= x_i <= 1 for i = 1,...,n.
+ *
+ * This should be optimized to x_i = 0.5 (for all x_i in x_M), at:
+ * 
+ * For more information, please refer to:
+ * 
+ * @code
+ * @incollection{deb2005scalable,
+ * title={Scalable test problems for evolutionary multiobjective optimization},
+ * author={Deb, Kalyanmoy and Thiele, Lothar and Laumanns, Marco and Zitzler, Eckart},
+ * booktitle={Evolutionary multiobjective optimization: theoretical advances and applications},
+ * pages={105--145},
+ * year={2005},
+ * publisher={Springer}
+ * }
+ * @endcode
+ *
+ * @tparam MatType Type of matrix to optimize.
+ */
+  template <typename MatType = arma::mat>
+  class DTLZ4
+  {
+    private:
+
+    // A fixed no. of Objectives and Variables(|x| = 7, M = 3).
+    size_t numObjectives {3};
+    size_t numVariables {7};
+    size_t numParetoPoints;
+    size_t alpha;
+
+    public:
+
+      /**
+       * Object Constructor.
+       * Initializes the individual objective functions.
+       *
+       * @param alpha The power which each variable is raised to.
+       * @param numParetoPoint No. of pareto points in the reference front.
+       */
+      DTLZ4 (size_t alpha = 100, size_t numParetoPoints = 136) :
+        alpha(alpha),
+        numParetoPoints(numParetoPoints),
+        objectiveF1(0, *this),
+        objectiveF2(1, *this),
+        objectiveF3(2, *this)
+      {/*Nothing to do here.*/}
+
+      //! Get the starting point.
+      arma::Col<typename MatType::elem_type> GetInitialPoint()
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+        return arma::Col<ElemType>(numVariables, 1, arma::fill::zeros);
+      } 
+      
+      // Get the private variables.
+
+      // Get the number of objectives.
+      size_t GetNumObjectives()
+      { return this -> numObjectives; }
+
+      // Get the number of variables.
+      size_t GetNumVariables()
+      { return this -> numVariables; }
+
+      /**
+       * Set the no. of pareto points.
+       *
+       * @param numParetoPoint
+       */
+      void SetNumParetoPoint(size_t numParetoPoint)
+      { this -> numParetoPoints = numParetoPoint; }
+
+      /**
+       * Evaluate the G(x) with the given coordinate.
+       *
+       * @param coords The function coordinates.
+       * @return arma::Row<typename MatType::elem_type>
+       */
+      arma::Row<typename MatType::elem_type> g(const MatType& coords)
+      {
+        size_t k = numVariables - numObjectives + 1;
+
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+        
+        arma::Row<ElemType> innerSum(size(coords)[1], arma::fill::zeros);
+        
+        for(size_t i = numObjectives - 1; i < numVariables; i++)
+        {
+          innerSum += arma::pow((coords.row(i) - 0.5), 2); 
+        } 
+        
+        return innerSum;
+      }    
+
+      /**
+       * Evaluate the objectives with the given coordinate.
+       *
+       * @param coords The function coordinates.
+       * @return arma::Mat<typename MatType::elem_type>
+       */
+      arma::Mat<typename MatType::elem_type> Evaluate(const MatType& coords)
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+
+        arma::Mat<ElemType> objectives(numObjectives, size(coords)[1]);
+        arma::Row<ElemType> G = g(coords);
+        arma::Row<ElemType> value = 0.5 * (1.0 + G);
+        for(size_t i = 0; i < numObjectives - 1; i++)
+        {
+          objectives.row(i) =  value %  
+              arma::sin(arma::pow(coords.row(i), alpha) * arma::datum::pi * 0.5);
+          value = value % arma::cos(arma::pow(coords.row(i), alpha) * arma::datum::pi * 0.5); 
+        }
+        objectives.row(numObjectives - 1) = value;
+        return objectives;    
+      }
+      
+      // Individual Objective function.
+      // Changes based on stop variable provided. 
+      struct DTLZ4Objective
+      {
+        DTLZ4Objective(size_t stop, DTLZ4& dtlz): stop(stop), dtlz(dtlz)
+        {/* Nothing to do here.*/}  
+        
+        /**
+         * Evaluate one objective with the given coordinate.
+         *
+         * @param coords The function coordinates.
+         * @return arma::Col<typename MatType::elem_type>
+         */
+        typename MatType::elem_type Evaluate(const MatType& coords)
+        {
+          // Convenience typedef.
+          typedef typename MatType::elem_type ElemType;
+          ElemType value = 1.0;
+          for(size_t i = 0; i < stop; i++)
+          {
+            value = value * std::cos(std::pow(coords[i], dtlz.alpha) * arma::datum::pi * 0.5);
+          }
+
+          if(stop != dtlz.numObjectives - 1)
+          {
+            value = value * std::sin(std::pow(coords[stop], dtlz.alpha) * arma::datum::pi * 0.5);
+          }
+          else
+          {
+            value = value * std::cos(std::pow(coords[stop], dtlz.alpha) * arma::datum::pi * 0.5);
+          }
+
+          value = value * (1 + dtlz.g(coords)[0]);
+          return value;  
+        }        
+
+        DTLZ4& dtlz;
+        size_t stop;
+      };
+
+      // Return back a tuple of objective functions.
+      std::tuple<DTLZ4Objective, DTLZ4Objective, DTLZ4Objective> GetObjectives()
+      {
+          return std::make_tuple(objectiveF1, objectiveF2, objectiveF3);
+      } 
+
+      //! Get the Reference Front.
+      //! Front. The implementation has been taken from pymoo.
+      arma::mat GetReferenceFront()
+      { 
+      	Uniform refGenerator;
+        arma::mat refDirs = refGenerator.Generate<arma::mat>(3, this -> numParetoPoints, 0);
+        arma::colvec x = arma::normalise(refDirs, 2, 1);
+        arma::mat A(size(refDirs), arma::fill::ones);
+        A.each_col() = x;
+        return refDirs / A;
+      }
+
+    DTLZ4Objective objectiveF1;
+    DTLZ4Objective objectiveF2;
+    DTLZ4Objective objectiveF3;
+  };
+  } //namespace test
+  } //namespace ens
+
+#endif
@@ -0,0 +1,210 @@
+/**
+ * @file dtlz5_function.hpp
+ * @author Satyam Shukla
+ *
+ * Implementation of the fifth DTLZ(Deb, Thiele, Laumanns, and Zitzler) test.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_DTLZ_FIVE_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_DTLZ_FIVE_FUNCTION_HPP
+
+namespace ens {
+namespace test {
+
+/**
+ * The DTLZ5 function, defined by:
+ * \f[
+ * theta_M = [theta_i, n - M + 1 <= i <= n]
+ * g(x) = \Sigma{i = n - M + 1}^n (x_i - 0.5)^2
+ * 
+ * f_1(x) = 0.5 * cos(theta_1 * pi * 0.5) * cos(theta_2 * pi * 0.5) * ... cos(theta_M-1 * pi * 0.5) * (1 + g(theta_M)) 
+ * f_2(x) = 0.5 * cos(theta_1 * pi * 0.5) * cos(theta_2 * pi * 0.5) * ... sin(theta_M-1 * pi * 0.5) * (1 + g(theta_M))
+ * .
+ * .
+ * f_M(x) = 0.5 * sin(theta_1 * pi * 0.5) * (1 + g(theta_M))
+ * \f]
+ *
+ * Bounds of the variable space is:
+ * 0 <= x_i <= 1 for i = 1,...,n.
+ * 
+ * Where theta_i = 0.5 * (1 + 2 * g(X_M) * x_i) / (1 + g(X_M))
+ * 
+ * This should be optimized to x_i = 0.5 (for all x_i in X_M), at:
+ * 
+ * For more information, please refer to:
+ * 
+ * @code
+ * @incollection{deb2005scalable,
+ * title={Scalable test problems for evolutionary multiobjective optimization},
+ * author={Deb, Kalyanmoy and Thiele, Lothar and Laumanns, Marco and Zitzler, Eckart},
+ * booktitle={Evolutionary multiobjective optimization: theoretical advances and applications},
+ * pages={105--145},
+ * year={2005},
+ * publisher={Springer}
+ * }
+ * @endcode
+ *
+ * @tparam MatType Type of matrix to optimize.
+ */
+  template <typename MatType = arma::mat>
+  class DTLZ5
+  {
+    private:
+
+    // A fixed no. of Objectives and Variables(|x| = 7, M = 3).
+    size_t numObjectives {3};
+    size_t numVariables {7};
+    size_t numParetoPoints;
+
+    public:
+
+      /**
+       * Object Constructor.
+       * Initializes the individual objective functions.
+       *
+       * @param numParetoPoint No. of pareto points in the reference front.
+       */
+      DTLZ5(size_t numParetoPoints = 136) :
+        numParetoPoints(numParetoPoints),
+        objectiveF1(0, *this),
+        objectiveF2(1, *this),
+        objectiveF3(2, *this)
+      {/*Nothing to do here.*/}
+
+      //! Get the starting point.
+      arma::Col<typename MatType::elem_type> GetInitialPoint()
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+        return arma::Col<ElemType>(numVariables, 1, arma::fill::zeros);
+      } 
+      
+      // Get the private variables.
+      
+      // Get the number of objectives.
+      size_t GetNumObjectives()
+      { return this -> numObjectives; }
+      
+      // Get the number of variables.
+      size_t GetNumVariables()
+      { return this -> numVariables; }
+
+      /**
+       * Set the no. of pareto points.
+       *
+       * @param numParetoPoint The no. points in the reference front.
+       */
+      void SetNumParetoPoint(size_t numParetoPoint)
+      { this -> numParetoPoints = numParetoPoint; }
+
+      /**
+       * Evaluate the G(x) with the given coordinate.
+       *
+       * @param coords The function coordinates.
+       * @return arma::Row<typename MatType::elem_type>
+       */
+      arma::Row<typename MatType::elem_type> g(const MatType& coords)
+      {
+        size_t k = numVariables - numObjectives + 1;
+
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+        
+        arma::Row<ElemType> innerSum(size(coords)[1], arma::fill::zeros);
+        
+        for(size_t i = numObjectives - 1; i < numVariables; i++)
+        {
+          innerSum += arma::pow((coords.row(i) - 0.5), 2); 
+        } 
+        
+        return innerSum;
+      }    
+
+      /**
+       * Evaluate the objectives with the given coordinate.
+       *
+       * @param coords The function coordinates.
+       * @return arma::Mat<typename MatType::elem_type>
+       */
+      arma::Mat<typename MatType::elem_type> Evaluate(const MatType& coords)
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+
+        arma::Mat<ElemType> objectives(numObjectives, size(coords)[1]);
+        arma::Row<ElemType> G = g(coords); 
+        arma::Row<ElemType> value = 0.5 * (1.0 + G);
+        arma::Row<ElemType> theta;
+        for(size_t i = 0; i < numObjectives - 1; i++)
+        {
+          theta = 0.5 * (1.0  + 2.0 * coords.row(i) % G) / (1.0 + G);
+          objectives.row(i) =  value %  
+              arma::sin(theta * arma::datum::pi * 0.5);
+          value = value % arma::cos(theta * arma::datum::pi * 0.5); 
+        }
+        objectives.row(numObjectives - 1) = value;
+        return objectives;
+      }
+      
+      // Individual Objective function.
+      // Changes based on stop variable provided. 
+      struct DTLZ5Objective
+      {
+        DTLZ5Objective(size_t stop, DTLZ5& dtlz): stop(stop), dtlz(dtlz)
+        {/* Nothing to do here. */}  
+        
+        /**
+         * Evaluate one objective with the given coordinate.
+         *
+         * @param coords The function coordinates.
+         * @return arma::Col<typename MatType::elem_type>
+         */
+        typename MatType::elem_type Evaluate(const MatType& coords)
+        {
+          // Convenience typedef.
+          typedef typename MatType::elem_type ElemType;
+          ElemType value = 1.0;
+          ElemType theta;
+          ElemType G = dtlz.g(coords)[0];
+          for(size_t i = 0; i < stop; i++)
+          {
+            theta = 0.5 * (1.0  + 2.0 * coords[i] * G) / (1.0 + G); 
+            value = value * std::cos(theta * arma::datum::pi * 0.5);
+          }
+	        theta = 0.5 * (1.0  + 2.0 * coords[stop] * G) / (1.0 + G);
+          if(stop != dtlz.numObjectives - 1)
+          {
+            value = value * std::sin(theta * arma::datum::pi * 0.5);
+          }
+          else
+          {
+            value = value * std::cos(theta * arma::datum::pi * 0.5);
+          }
+
+          value = value * (1.0 + G);
+          return value;  
+        }        
+
+        DTLZ5& dtlz;
+        size_t stop;
+      };
+
+      // Return back a tuple of objective functions.
+      std::tuple<DTLZ5Objective, DTLZ5Objective, DTLZ5Objective> GetObjectives ()
+      {
+          return std::make_tuple(objectiveF1, objectiveF2, objectiveF3);
+      } 
+
+    DTLZ5Objective objectiveF1;
+    DTLZ5Objective objectiveF2;
+    DTLZ5Objective objectiveF3;
+  };
+  } //namespace test
+  } //namespace ens
+
+#endif
@@ -0,0 +1,210 @@
+/**
+ * @file dtlz6_function.hpp
+ * @author Satyam Shukla
+ *
+ * Implementation of the sixth DTLZ(Deb, Thiele, Laumanns, and Zitzler) test.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_DTLZ_SIX_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_DTLZ_SIX_FUNCTION_HPP
+
+namespace ens {
+namespace test {
+
+/**
+ * The DTLZ5 function, defined by:
+ * \f[
+ * theta_M = [theta_i, n - M + 1 <= i <= n]
+ * g(x) = \Sigma{i = n - M + 1}^n (x_i)^0.1
+ * 
+ * f_1(x) = 0.5 * cos(theta_1 * pi * 0.5) * cos(theta_2 * pi * 0.5) * ... cos(theta_2 * pi * 0.5) * (1 + g(theta_M)) 
+ * f_2(x) = 0.5 * cos(theta_1 * pi * 0.5) * cos(theta_2 * pi * 0.5) * ... sin(theta_M-1 * pi * 0.5) * (1 + g(theta_M))
+ * .
+ * .
+ * f_M(x) = 0.5 * sin(theta_1 * pi * 0.5) * (1 + g(theta_M))
+ * \f]
+ *
+ * Bounds of the variable space is:
+ * 0 <= x_i <= 1 for i = 1,...,n.
+ * 
+ * Where theta_i = 0.5 * (1 + 2 * g(X_M) * x_i) / (1 + g(X_M))
+ * 
+ * This should be optimized to x_i = 0.5 (for all x_i in X_M), at:
+ * 
+ * For more information, please refer to:
+ * 
+ * @code
+ * @incollection{deb2005scalable,
+ * title={Scalable test problems for evolutionary multiobjective optimization},
+ * author={Deb, Kalyanmoy and Thiele, Lothar and Laumanns, Marco and Zitzler, Eckart},
+ * booktitle={Evolutionary multiobjective optimization: theoretical advances and applications},
+ * pages={105--145},
+ * year={2005},
+ * publisher={Springer}
+ * }
+ * @endcode
+ *
+ * @tparam MatType Type of matrix to optimize.
+ */
+  template <typename MatType = arma::mat>
+  class DTLZ6
+  {
+    private:
+
+    // A fixed no. of Objectives and Variables(|x| = 7, M = 3).
+    size_t numObjectives {3};
+    size_t numVariables {7};
+    size_t numParetoPoints;
+
+    public:
+
+      /**
+       * Object Constructor.
+       * Initializes the individual objective functions.
+       *
+       * @param numParetoPoint No. of pareto points in the reference front.
+       */
+      DTLZ6(size_t numParetoPoints = 136) :
+        numParetoPoints(numParetoPoints),
+        objectiveF1(0, *this),
+        objectiveF2(1, *this),
+        objectiveF3(2, *this)
+      {/*Nothing to do here.*/}
+
+      //! Get the starting point.
+      arma::Col<typename MatType::elem_type> GetInitialPoint ()
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+        return arma::Col<ElemType>(numVariables, 1, arma::fill::zeros);
+      } 
+      
+      // Get the private variables.
+
+      // Get number of obectives.
+      size_t GetNumObjectives()
+      { return this -> numObjectives; }
+
+      // Get number of variables.
+      size_t GetNumVariables()
+      { return this -> numVariables; }
+
+      /**
+       * Set the no. of pareto points.
+       *
+       *  @param numParetoPoint No. of pareto points in the reference front.
+       */
+      void SetNumParetoPoint(size_t numParetoPoint)
+      { this -> numParetoPoints = numParetoPoint; }
+
+      /**
+       * Evaluate the G(x) with the given coordinate.
+       *
+       * @param coords The function coordinates.
+       * @return arma::Row<typename MatType::elem_type>
+       */
+      arma::Row<typename MatType::elem_type> g(const MatType& coords)
+      {
+        size_t k = numVariables - numObjectives + 1;
+
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+        
+        arma::Row<ElemType> innerSum(size(coords)[1], arma::fill::zeros);
+        
+        for(size_t i = numObjectives - 1; i < numVariables; i++)
+        {
+          innerSum += arma::pow(coords.row(i), 0.1); 
+        }
+        
+        return innerSum;
+      }    
+
+      /**
+       * Evaluate the objectives with the given coordinate.
+       *
+       * @param coords The function coordinates.
+       * @return arma::Mat<typename MatType::elem_type>
+       */
+      arma::Mat<typename MatType::elem_type> Evaluate(const MatType& coords)
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+
+        arma::Mat<ElemType> objectives(numObjectives, size(coords)[1]);
+        arma::Row<ElemType> G = g(coords); 
+        arma::Row<ElemType> value = 0.5 * (1.0 + G);
+        arma::Row<ElemType> theta;
+        for(size_t i = 0; i < numObjectives - 1; i++)
+        {
+          theta = 0.5 * (1.0  + 2.0 * coords.row(i) % G) / (1.0 + G);
+          objectives.row(i) =  value %  
+              arma::sin(theta * arma::datum::pi * 0.5);
+          value = value % arma::cos(theta * arma::datum::pi * 0.5); 
+        }
+        objectives.row(numObjectives - 1) = value;
+        return objectives;
+      }
+      
+      // Individual Objective function.
+      // Changes based on stop variable provided. 
+      struct DTLZ6Objective
+      {
+        DTLZ6Objective(size_t stop, DTLZ6& dtlz): stop(stop), dtlz(dtlz)
+        {/* Nothing to do here. */}  
+        
+        /**
+         * Evaluate one objective with the given coordinate.
+         *
+         * @param coords The function coordinates.
+         * @return arma::Col<typename MatType::elem_type>
+         */
+        typename MatType::elem_type Evaluate(const MatType& coords)
+        {
+          // Convenience typedef.
+          typedef typename MatType::elem_type ElemType;
+          ElemType value = 1.0;
+          ElemType theta;
+          ElemType G = dtlz.g(coords)[0];
+          for(size_t i = 0; i < stop; i++)
+          {
+            theta = 0.5 * (1.0  + 2.0 * coords[i] * G) / (1.0 + G); 
+            value = value * std::cos(theta * arma::datum::pi * 0.5);
+          }
+	        theta = 0.5 * (1.0  + 2.0 * coords[stop] * G) / (1.0 + G);
+          if(stop != dtlz.numObjectives - 1)
+          {
+            value = value * std::sin(theta * arma::datum::pi * 0.5);
+          }
+          else
+          {
+            value = value * std::cos(theta * arma::datum::pi * 0.5);
+          }
+
+          value = value * (1.0 + G);
+          return value;  
+        }        
+
+        DTLZ6& dtlz;
+        size_t stop;
+      };
+
+      // Return back a tuple of objective functions.
+      std::tuple<DTLZ6Objective, DTLZ6Objective, DTLZ6Objective> GetObjectives ()
+      {
+          return std::make_tuple(objectiveF1, objectiveF2, objectiveF3);
+      } 
+
+    DTLZ6Objective objectiveF1;
+    DTLZ6Objective objectiveF2;
+    DTLZ6Objective objectiveF3;
+  };
+  } //namespace test
+  } //namespace ens
+
+#endif
@@ -0,0 +1,214 @@
+/**
+ * @file dtlz7_function.hpp
+ * @author Satyam Shukla
+ *
+ * Implementation of the seventh DTLZ(Deb, Thiele, Laumanns, and Zitzler) test.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_DTLZ_SEVEN_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_DTLZ_SEVEN_FUNCTION_HPP
+
+namespace ens {
+namespace test {
+
+/**
+ * The DTLZ7 function, defined by:
+ * \f[
+ * x_M = [x_i, n - M + 1 <= i <= n]
+ * g(x) = 1 + (9 / |X_M|) * (\Sigma{i = n - M + 1}^n x_i) 
+ * 
+ * f_1(x) = x_1
+ * f_2(x) = x_2
+ * .
+ * .
+ * f_M(x) = (1 + g(X_M)) * h(f_1, f_2,...,g)
+ * \f]
+ *
+ * Bounds of the variable space is:
+ * 0 <= x_i <= 1 for i = 1,...,n.
+ *
+ * This should be optimized to x_i = 0.5 (for all x_i in x_M), at:
+ * the objective function values lie on the linear hyper-plane: 
+ * \Sigma { m = 1}^M f_m* =0.5.
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @incollection{deb2005scalable,
+ * title={Scalable test problems for evolutionary multiobjective optimization},
+ * author={Deb, Kalyanmoy and Thiele, Lothar and Laumanns, Marco and Zitzler, Eckart},
+ * booktitle={Evolutionary multiobjective optimization: theoretical advances and applications},
+ * pages={105--145},
+ * year={2005},
+ * publisher={Springer}
+ * }
+ * @endcode
+ *
+ * @tparam MatType Type of matrix to optimize.
+ */
+  template <typename MatType = arma::mat>
+  class DTLZ7
+  {
+    private:
+
+    // A fixed no. of Objectives and Variables(|x| = 7, M = 3).
+    size_t numObjectives {3};
+    size_t numVariables {7};
+    size_t numParetoPoints;
+
+    public:
+
+      /**
+       * Object Constructor.
+       * Initializes the individual objective functions.
+       *
+       * @param numParetoPoint No. of pareto points in the reference front.
+       */
+      DTLZ7(size_t numParetoPoint = 136) :
+        numParetoPoints(numParetoPoint),
+        objectiveF1(0, *this),
+        objectiveF2(1, *this),
+        objectiveF3(2, *this)
+      {/* Nothing to do here */}
+
+      // Get the private variables.
+
+      // Get the number of objectives.
+      size_t GetNumObjectives()
+      { return this -> numObjectives; }
+
+      // Get the number of variables.
+      size_t GetNumVariables()
+      { return this -> numVariables; }
+
+      /**
+       * Set the no. of pareto points.
+       *
+       * @param numParetoPoint No. of pareto points in the reference front.
+       */
+      void SetNumParetoPoint(size_t numParetoPoint)
+      { this -> numParetoPoints = numParetoPoint; }
+
+      // Get the starting point.
+      arma::Col<typename MatType::elem_type> GetInitialPoint()
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+        return arma::Col<ElemType>(numVariables, arma::fill::zeros);
+      } 
+
+      /**
+       * Evaluate the G(x) with the given coordinate.
+       *
+       * @param coords The function coordinates.
+       * @return arma::Row<typename MatType::elem_type>
+       */
+      arma::Row<typename MatType::elem_type> g(const MatType& coords)
+      {
+        size_t k = numVariables - numObjectives + 1;
+
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+        
+        arma::Row<ElemType> innerSum(size(coords)[1], arma::fill::zeros);
+        
+        innerSum = (9.0 / k) * arma::sum(coords.rows(numObjectives - 1,
+            numVariables - 1) , 0) + 1.0; 
+        return innerSum;
+      }
+
+      /**
+       * Evaluate the H(f_i,...) with the given coordinate.
+       *
+       * @param coords The function coordinates.
+       * @return arma::Row<typename MatType::elem_type>
+       */
+      arma::Row<typename MatType::elem_type> h(
+          const MatType& coords, const arma::Row<typename MatType::elem_type>& G)
+      {
+        size_t k = numVariables - numObjectives + 1;
+
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+        
+        arma::Row<ElemType> innerSum(size(coords)[1], arma::fill::ones);
+        innerSum = innerSum * numObjectives;
+        for(size_t i = 0;i < numObjectives - 1;i++)
+        {
+            innerSum -= coords.row(i) % (1.0 + 
+                    arma::cos(arma::datum::pi * 3 * coords.row(i))) / (1 + G); 
+        }
+        return innerSum;
+      }
+
+      /**
+       * Evaluate the objectives with the given coordinate.
+       *
+       * @param coords The function coordinates.
+       * @return arma::Mat<typename MatType::elem_type>
+       */
+      arma::Mat<typename MatType::elem_type> Evaluate(const MatType& coords)
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+
+        arma::Mat<ElemType> objectives(numObjectives, size(coords)[1]); 
+        arma::Row<ElemType> G = g(coords);
+        arma::Row<ElemType> H = h(coords, G);
+        objectives.rows(0, numObjectives - 2) = coords.rows(0, numObjectives - 2);
+        objectives.row(numObjectives - 1) = (1 + G) % H;
+        return objectives;    
+      }
+      
+      // Individual Objective function.
+      // Changes based on stop variable provided. 
+      struct DTLZ7Objective
+      {
+        DTLZ7Objective(size_t stop, DTLZ7& dtlz): stop(stop), dtlz(dtlz)
+        {/* Nothing to do here. */}  
+        
+        /**
+         * Evaluate one objective with the given coordinate.
+         *
+         * @param coords The function coordinates.
+         * @return arma::Col<typename MatType::elem_type>
+         */
+        typename MatType::elem_type Evaluate(const MatType& coords)
+        {
+          // Convenience typedef.
+          typedef typename MatType::elem_type ElemType;
+          ElemType value = 0.5;
+          if(stop != dtlz.numObjectives - 1)
+          { return coords[stop];}
+ 
+          value = (1.0 + dtlz.g(coords)[0]) * dtlz.h(coords, dtlz.g(coords))[0];
+          return value; 
+        }        
+
+        DTLZ7& dtlz;
+        size_t stop;
+      };
+
+      // Return back a tuple of objective functions.
+      std::tuple<DTLZ7Objective, DTLZ7Objective, DTLZ7Objective> GetObjectives()
+      {
+          return std::make_tuple(objectiveF1, objectiveF2, objectiveF3);
+      } 
+
+    DTLZ7Objective objectiveF1;
+    DTLZ7Objective objectiveF2;
+    DTLZ7Objective objectiveF3;
+    using MAF7Objective = DTLZ7Objective;
+  };
+
+  template<typename MatType>
+  using MAF7 = DTLZ7<MatType>;
+  } //namespace test
+  } //namespace ens
+
+#endif
@@ -0,0 +1,188 @@
+/**
+ * @file maf1_function.hpp
+ * @author Satyam Shukla
+ *
+ * Implementation of the first Maf test.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_MAF_ONE_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_MAF_ONE_FUNCTION_HPP
+
+#include "../../moead/weight_init_policies/uniform_init.hpp"
+
+namespace ens {
+namespace test {
+
+/**
+ * The MAF1 function, defined by:
+ * \f[
+ * x_M = [x_i, n - M + 1 <= i <= n]
+ * g(x) = \Sigma{i = n - M + 1}^n (x_i - 0.5)^2
+ * 
+ * f_1(x) = 1 - x_1 * x_2 * ... x_M-1 * (1 + g(x_M)) 
+ * f_2(x) = 1 - x_1 * x_2 * ... (1 - x_M-1) * (1 + g(x_M))
+ * .
+ * .
+ * f_M(x) = (x_1) * (1 + g(x_M))
+ * \f]
+ *
+ * Bounds of the variable space is:
+ * 0 <= x_i <= 1 for i = 1,...,n.
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{cheng2017benchmark,
+ * title={A benchmark test suite for evolutionary many-objective optimization},
+ * author={Cheng, Ran and Li, Miqing and Tian, Ye and Zhang, Xingyi and Yang, Shengxiang and Jin, Yaochu and Yao, Xin},
+ * journal={Complex \& Intelligent Systems},
+ * volume={3},
+ * pages={67--81},
+ * year={2017},
+ * publisher={Springer}
+ * }
+ * @endcode
+ *
+ * @tparam MatType Type of matrix to optimize.
+ */
+  template <typename MatType = arma::mat>
+  class MAF1
+  {
+    private:
+
+    // A fixed no. of Objectives and Variables(|x| = 7, M = 3).
+    size_t numObjectives {3};
+    size_t numVariables {12};
+
+    public:
+
+      /**
+      * Object Constructor.
+      * Initializes the individual objective functions.
+      *
+      * @param numParetoPoint No. of pareto points in the reference front.
+      */
+      MAF1() :
+          objectiveF1(0, *this),
+          objectiveF2(1, *this),
+          objectiveF3(2, *this)
+      {/* Nothing to do here */}
+
+      // Get the private variables.
+      size_t GetNumObjectives()
+      { return this -> numObjectives; }
+
+      size_t GetNumVariables()
+      { return this -> numVariables; }
+
+      // Get the starting point.
+      arma::Col<typename MatType::elem_type> GetInitialPoint()
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+        return arma::Col<ElemType>(numVariables, arma::fill::ones);
+      }
+
+      /**
+      * Evaluate the G(x) with the given coordinate.
+      *
+      * @param coords The function coordinates.
+      * @return arma::Row<typename MatType::elem_type>
+      */
+      arma::Row<typename MatType::elem_type> g(const MatType& coords)
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+        
+        arma::Row<ElemType> innerSum(size(coords)[1], arma::fill::zeros);
+        
+        for (size_t i = numObjectives - 1;i < numVariables;i++)
+        {
+          innerSum += arma::pow((coords.row(i) - 0.5), 2);
+        } 
+        
+        return innerSum;
+      }     
+
+      /**
+      * Evaluate the objectives with the given coordinate.
+      *
+      * @param coords The function coordinates.
+      * @return arma::Mat<typename MatType::elem_type>
+      */
+      arma::Mat<typename MatType::elem_type> Evaluate(const MatType& coords)
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+
+        arma::Mat<ElemType> objectives(numObjectives, size(coords)[1]);
+        arma::Row<ElemType> G = g(coords);
+        arma::Row<ElemType> value(coords.n_cols, arma::fill::ones);
+        for (size_t i = 0;i < numObjectives - 1;i++)
+        {
+          objectives.row(i) = (1 - value % (1.0 - coords.row(i))) % (1. + G);
+          value = value % coords.row(i);
+        }
+        objectives.row(numObjectives - 1) = (1 - value) % (1. + G);
+        return objectives;    
+      }
+      
+      // Individual Objective function.
+      // Changes based on stop variable provided. 
+      struct MAF1Objective
+      {
+        MAF1Objective(size_t stop, MAF1& maf): stop(stop), maf(maf)
+        {/* Nothing to do here. */}  
+        
+        /**
+        * Evaluate one objective with the given coordinate.
+        *
+        * @param coords The function coordinates.
+        * @return arma::Col<typename MatType::elem_type>
+        */
+        typename MatType::elem_type Evaluate(const MatType& coords)
+        {
+          // Convenience typedef.
+          if(stop == 0)
+          {
+            return coords[0] * (1. + maf.g(coords)[0]);
+          }
+          typedef typename MatType::elem_type ElemType;
+          ElemType value = 1.0;
+          for (size_t i = 0; i < stop; i++)
+          {
+            value = value * coords[i];
+          }
+
+          if(stop != maf.GetNumObjectives() - 1)
+          {
+            value = value * (1. - coords[stop]);
+          }
+
+          value = (1.0 - value) * (1. + maf.g(coords)[0]);
+          return value;
+        }        
+
+        MAF1& maf;
+        size_t stop;
+      };
+
+      //! Get objective functions.
+      std::tuple<MAF1Objective, MAF1Objective, MAF1Objective> GetObjectives()
+      {
+        return std::make_tuple(objectiveF1, objectiveF2, objectiveF3);
+      }
+
+      MAF1Objective objectiveF1;
+      MAF1Objective objectiveF2;
+      MAF1Objective objectiveF3;
+  };
+  } //namespace test
+  } //namespace ens
+
+#endif
--- a/Show More
+++ b/Show More