Update and release version 2.18.2.

Accelerate SGD test time by skipping additional trials.

.appveyor.yml

@@ -0,0 +1,57 @@
+environment:
+  ARMADILLO_DOWNLOAD: "https://sourceforge.net/projects/arma/files/armadillo-9.800.6.tar.xz"
+  BLAS_LIBRARY: "%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a"
+  BLAS_LIBRARY_DLL: "%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll"
+
+  matrix:
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
+      VSVER: Visual Studio 14 2015 Win64
+      MSBUILD: C:\Program Files (x86)\MSBuild\14.0\bin\MSBuild.exe
+
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+      VSVER: Visual Studio 15 2017 Win64
+      MSBUILD: C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\MSBuild.exe
+
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
+      VSVER: Visual Studio 16 2019
+      MSBUILD: C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe
+
+configuration: Release
+
+install:
+  - ps: nuget install OpenBLAS -o "${env:APPVEYOR_BUILD_FOLDER}"
+
+build_script:
+  # First, download and build Armadillo.
+  - cd ..
+  - appveyor DownloadFile %ARMADILLO_DOWNLOAD% -FileName armadillo.tar.xz
+  - 7z x armadillo.tar.xz -so -txz | 7z x -si -ttar > nul
+  - cd armadillo-9.800.6 && mkdir build && cd build
+  - >
+    cmake -G "%VSVER%"
+    -DBLAS_LIBRARY:FILEPATH=%BLAS_LIBRARY%
+    -DLAPACK_LIBRARY:FILEPATH=%BLAS_LIBRARY%
+    -DCMAKE_PREFIX:FILEPATH="%APPVEYOR_BUILD_FOLDER%/armadillo"
+    -DBUILD_SHARED_LIBS=OFF
+    -DCMAKE_BUILD_TYPE=Release ..
+  - >
+    "%MSBUILD%" "armadillo.sln"
+    /m /verbosity:quiet /p:Configuration=Release;Platform=x64
+  - cd ../..
+
+  # Now build ensmallen.
+  - cd ensmallen && mkdir build && cd build
+  - >
+    cmake -G "%VSVER%"
+    -DARMADILLO_INCLUDE_DIR=%APPVEYOR_BUILD_FOLDER%/../armadillo-9.800.6/include/
+    -DARMADILLO_LIBRARIES=%BLAS_LIBRARY%
+    -DLAPACK_LIBRARY=%BLAS_LIBRARY%
+    -DBLAS_LIBRARY=%BLAS_LIBRARY%
+    -DCMAKE_BUILD_TYPE=Release ..
+  - >
+    "%MSBUILD%" "ensmallen.sln"
+    /m /verbosity:minimal /nologo /p:BuildInParallel=true
+
+  # Run tests after copying libraries.
+  - ps: cp C:\projects\ensmallen\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.* C:\projects\ensmallen\build\
+  - ctest -C Release -V --output-on-failure .

.travis.yml

@@ -1,26 +1,32 @@
-sudo: required
-dist: trusty
+os: linux
+dist: focal
 language: cpp
 
 env:
-  - ARMADILLO=latest
+  - ARMADILLO=latest SANITY_HISTORY=perform
   - ARMADILLO=minimum
 
-before_install:
+stages:
+  - test
+  - name: sanity
+    if: type = pull_request AND env(SANITY_HISTORY) = "perform"
+
+jobs:
+  include:
+    - stage: sanity
+      name: "HISTORY.md Check"
+      script: sh ./scripts/history-update-check.sh
+
+script:
   - sudo apt-get update
   - sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ xz-utils
   - if [ $ARMADILLO == "latest" ]; then
       curl https://ftp.fau.de/macports/distfiles/armadillo/`curl https://ftp.fau.de/macports/distfiles/armadillo/ -- | grep '.tar.xz' | sed 's/^.*<a href="\(armadillo-[0-9]*.[0-9]*.[0-9]*.tar.xz\)".*$/\1/' | tail -1` | tar xvJ && cd armadillo*;
-    fi
-  - if [ $ARMADILLO == "minimum" ]; then
-      curl https://ftp.fau.de/macports/distfiles/armadillo/armadillo-8.400.0.tar.xz | tar -xvJ && cd armadillo*;
+    else
+      curl -L https://sourceforge.net/projects/arma/files/armadillo-9.800.6.tar.xz | tar -xvJ && cd armadillo*;
     fi
   - cmake . && make && sudo make install && cd ..
-
-install:
-  - mkdir build && cd build && cmake -DCMAKE_CXX_FLAGS="-Werror" -DCMAKE_C_FLAGS="-Werror" .. && make -j2
-
-script:
+  - mkdir build && cd build && cmake .. && make ensmallen_tests -j2
   - CTEST_OUTPUT_ON_FAILURE=1 travis_wait 30 ctest -j2
 
 notifications:

CMake/FindArmadillo.cmake

@@ -366,4 +366,10 @@ mark_as_advanced(
   ARMADILLO_INCLUDE_DIR
   ARMADILLO_LIBRARIES)
 
+if (ARMADILLO_FOUND AND NOT TARGET Armadillo:Armadillo)
+  add_library(Armadillo::Armadillo INTERFACE IMPORTED)
+  set_target_properties(Armadillo::Armadillo PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${ARMADILLO_INCLUDE_DIR}"
+      INTERFACE_LINK_LIBRARIES "${ARMADILLO_LIBRARIES}")
+endif()
+
 #======================

CMake/ensmallen-config.cmake.in

@@ -0,0 +1,4 @@
+@PACKAGE_INIT@
+
+include(${CMAKE_CURRENT_LIST_DIR}/@TARGETS_EXPORT_NAME@.cmake)
+check_required_components(ensmallen)

CMakeLists.txt

@@ -1,63 +1,91 @@
 # ensmallen CMake configuration.  This project has no configurable options---it
 # just installs the headers to the install location, and optionally builds the
 # test program.
-cmake_minimum_required(VERSION 2.8.10)
-project(ensmallen C CXX)
+cmake_minimum_required(VERSION 3.3.2)
+project(ensmallen
+        LANGUAGES C CXX)
 
+# Configurable options for CMake.
 option(USE_OPENMP "If available, use OpenMP for parallelization." ON)
-option(BUILD_TESTS "Build tests." ON)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/CMake")
 
-# Ensure we have C++11 features.  Since we support CMake < 3.1, this needs a
-# little bit of special handling.
-if ((${CMAKE_MAJOR_VERSION} LESS 3 OR
-    (${CMAKE_MAJOR_VERSION} EQUAL 3 AND ${CMAKE_MINOR_VERSION} LESS 1))
-    AND NOT FORCE_CXX11)
-  # Older versions of CMake do not support target_compile_features(), so we have
-  # to use something kind of hacky.
-  include(CMake/CXX11.cmake)
-  check_for_cxx11_compiler(HAS_CXX11)
-  if(NOT HAS_CXX11)
-    message(FATAL_ERROR "No C++11 compiler available!")
-  endif()
-  enable_cxx11()
-else()
-  # set required standard to c++11
-  set(CMAKE_CXX_STANDARD 11)
-  set(CMAKE_CXX_STANDARD_REQUIRED ON)
-endif ()
+# Set required C++ standard to C++11.
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-# Detect OpenMP support in a compiler. If the compiler supports OpenMP, flags
-# to compile with OpenMP are returned and added for compilation.
-if (USE_OPENMP)
-  find_package(OpenMP)
-endif ()
+# Extract version from sources.
+set(ENSMALLEN_VERSION_FILE_NAME "${PROJECT_SOURCE_DIR}/include/ensmallen_bits/ens_version.hpp")
 
-if (OPENMP_FOUND)
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-else ()
-  # Disable warnings for all the unknown OpenMP pragmas.
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-pragmas")
-endif ()
-
-# Set the CFLAGS and CXXFLAGS depending on the options the user specified.
-if(CMAKE_COMPILER_IS_GNUCC OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
-
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wpedantic -Wunused-parameter")
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wpedantic -Wunused-parameter")
+if(NOT EXISTS ${ENSMALLEN_VERSION_FILE_NAME})
+  message(FATAL_ERROR "Can't read ${ENSMALLEN_VERSION_FILE_NAME}")
 endif()
 
-# The only dependency we need is Armadillo.
-#
-# We keep the minimum version in sync with mlpack, otherwise we could have
-# irritating compatibility issues.
-find_package(Armadillo 8.400.0 REQUIRED)
-include_directories(BEFORE "${ARMADILLO_INCLUDE_DIR}")
-include_directories(BEFORE "${CMAKE_SOURCE_DIR}/include/")
+file(READ ${ENSMALLEN_VERSION_FILE_NAME} ENSMALLEN_VERSION_FILE_CONTENTS)
+string(REGEX REPLACE ".*#define ENS_VERSION_MAJOR ([0-9]+).*" "\\1" ENSMALLEN_VERSION_MAJOR "${ENSMALLEN_VERSION_FILE_CONTENTS}")
+string(REGEX REPLACE ".*#define ENS_VERSION_MINOR ([0-9]+).*" "\\1" ENSMALLEN_VERSION_MINOR "${ENSMALLEN_VERSION_FILE_CONTENTS}")
+string(REGEX REPLACE ".*#define ENS_VERSION_PATCH ([0-9]+).*" "\\1" ENSMALLEN_VERSION_PATCH "${ENSMALLEN_VERSION_FILE_CONTENTS}")
 
-# Install the headers to the correct location.
+message(STATUS "Configuring ensmallen ${ENSMALLEN_VERSION_MAJOR}.${ENSMALLEN_VERSION_MINOR}.${ENSMALLEN_VERSION_PATCH}")
+set(VERSION "${ENSMALLEN_VERSION_MAJOR}.${ENSMALLEN_VERSION_MINOR}.${ENSMALLEN_VERSION_PATCH}")
+
+# Create library target.
+add_library(ensmallen INTERFACE)
+target_include_directories(ensmallen INTERFACE
+  $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
+  $<INSTALL_INTERFACE:include>)
+
+# Set warning flags for target.
+if(MSVC)
+  target_compile_options(ensmallen INTERFACE $<BUILD_INTERFACE:/Wall>)
+else()
+  target_compile_options(ensmallen INTERFACE $<BUILD_INTERFACE:-Wall -Wpedantic -Wunused-parameter>)
+endif()
+
+# Find OpenMP and link it.
+if(USE_OPENMP)
+  if(NOT TARGET OpenMP::OpenMP_CXX)
+    find_package(Threads REQUIRED)
+    add_library(OpenMP::OpenMP_CXX IMPORTED INTERFACE)
+    set_property(TARGET OpenMP::OpenMP_CXX
+        PROPERTY INTERFACE_COMPILE_OPTIONS ${OpenMP_CXX_FLAGS})
+    # Only works if the same flag is passed to the linker; use CMake 3.9+ otherwise (Intel, AppleClang).
+    set_property(TARGET OpenMP::OpenMP_CXX
+        PROPERTY INTERFACE_LINK_LIBRARIES ${OpenMP_CXX_FLAGS} Threads::Threads)
+  endif()
+  target_link_libraries(ensmallen INTERFACE OpenMP::OpenMP_CXX)
+endif()
+
+# Find Armadillo and link it.
+find_package(Armadillo 9.800.0 REQUIRED)
+target_link_libraries(ensmallen INTERFACE Armadillo::Armadillo)
+
+# Set helper variables for creating the version, config and target files.
+include(CMakePackageConfigHelpers)
+set(ENSMALLEN_CMAKE_DIR "lib/cmake/ensmallen" CACHE STRING
+      "Installation directory for cmake files, relative to ${CMAKE_INSTALL_PREFIX}.")
+set(VERSION_CONFIG "${PROJECT_BINARY_DIR}/ensmallen-config-version.cmake")
+set(PROJECT_CONFIG "${PROJECT_BINARY_DIR}/ensmallen-config.cmake")
+set(TARGETS_EXPORT_NAME ensmallen-targets)
+
+# Generate the version, config and target files into the build directory.
+write_basic_package_version_file(${VERSION_CONFIG}
+    VERSION ${VERSION}
+    COMPATIBILITY AnyNewerVersion)
+configure_package_config_file(${PROJECT_SOURCE_DIR}/CMake/ensmallen-config.cmake.in
+    ${PROJECT_CONFIG}
+    INSTALL_DESTINATION ${ENSMALLEN_CMAKE_DIR})
+export(TARGETS ensmallen NAMESPACE ensmallen::
+    FILE ${PROJECT_BINARY_DIR}/${TARGETS_EXPORT_NAME}.cmake)
+
+# Install version, config and target files.
+install(FILES ${PROJECT_CONFIG} ${VERSION_CONFIG}
+    DESTINATION ${ENSMALLEN_CMAKE_DIR})
+install(EXPORT ${TARGETS_EXPORT_NAME} DESTINATION ${ENSMALLEN_CMAKE_DIR}
+    NAMESPACE ensmallen::)
+
+# Export the targets and install the header files.
+install(TARGETS ensmallen EXPORT ${TARGETS_EXPORT_NAME} DESTINATION lib)
 install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/ensmallen_bits"
         DESTINATION "${CMAKE_INSTALL_PREFIX}/include"
         PATTERN "*~" EXCLUDE
@@ -65,8 +93,6 @@ install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/ensmallen_bits"
 install(FILES ${CMAKE_SOURCE_DIR}/include/ensmallen.hpp
         DESTINATION "${CMAKE_INSTALL_PREFIX}/include")
 
+# Enable testing and build tests.
 enable_testing()
-
-if (BUILD_TESTS)
-  add_subdirectory(tests)
-endif()
+add_subdirectory(tests)

CONTRIBUTING.md

@@ -22,7 +22,7 @@ merge, to ensure that:
 
 Please do make sure that if you contribute a new optimizer or other new
 functionality, that you've added some tests in the `tests/` directory.  And if
-you are fixing a bug, it's always nice to include a test case if possible to so
+you are fixing a bug, it's always nice to include a test case if possible so
 that the bug won't happen again.
 
 ## Build/test process
@@ -42,7 +42,116 @@ you have written a test like this, make sure it does not fail often by
 uncommenting the code that sets a random seed in `tests/main.cpp` and running
 your test many times.
 
-Information on how to build and run the tests is in the main README.md file.
+### Install Dependencies
+
+<details open>
+<summary>Linux</summary>
+
+Use your distributions' package manager to install the required dependencies. For Ubuntu/Debian, the commands are as shown below.
+
+```bash
+$ sudo apt-get update
+$ sudo apt-get install libarmadillo-dev cmake
+```
+
+To install the dependencies on Fedora/RHEL/CentOS, you can use:
+
+```bash
+$ yum install armadillo-devel cmake
+```
+
+You can install **ensmallen** on Arch from AUR using:
+
+```bash
+$ git clone https://aur.archlinux.org/ensmallen.git
+$ cd ensmallen
+$ makepkg -si
+```
+
+</details>
+
+<details open>
+<summary>macOS</summary>
+
+You can install **ensmallen** directly with [Homebrew](https://brew.sh).
+
+```bash
+$ brew install ensmallen
+```
+
+If you want to build **ensmallen** tests from source, use a package manager like [Homebrew](https://brew.sh) to get the necessary dependencies.
+
+```bash
+$ brew install armadillo cmake
+```
+</details>
+
+<details open>
+<summary>Windows</summary>
+
+You can install **ensmallen** directly by using [vcpkg](https://github.com/microsoft/vcpkg).
+
+```
+vcpkg install ensmallen:x64-windows
+```
+</details>
+
+### Build and Test
+
+This section describes how to build the **ensmallen** tests from source. **ensmallen** uses CMake as its build system and [Catch2](https://github.com/catchorg/Catch2) as the unit test framework.
+
+First, clone the source code from Github and change into the cloned directory. Alternatively, you can download the latest release from the [website](http://ensmallen.org) and extract it.
+
+```bash
+$ git clone https://github.com/mlpack/ensmallen
+$ cd ensmallen
+
+# - or -
+
+$ wget http://ensmallen.org/files/ensmallen-2.18.2.tar.gz
+$ tar -xvzpf ensmallen-2.18.2.tar.gz
+$ cd ensmallen-latest
+```
+
+Next, make a build directory and change into that directory.
+
+```bash
+$ mkdir build
+$ cd build
+```
+
+Then, run the cmake command followed by the make command in the build directory. If the cmake command fails, you probably have missing dependencies.
+
+```bash
+$ cmake ..
+# or with -w flag to inhibit all warning messages
+# $ cmake -DCMAKE_CXX_FLAGS="-w" -DCMAKE_C_FLAGS="-w" .. #
+$ make
+# or alternately make with the -jN flag to run parallel jobs
+# $ make -j4
+```
+
+Now, you can either run all of the tests or an individual test case with:
+
+```bash
+$ ./ensmallen_tests
+$ ./ensmallen_tests <test name>
+```
+
+You can list all tests with:
+
+```bash
+$ ./ensmallen-tests -l
+
+ensmallen version: 2.10.5 (Fried Chicken)
+armadillo version: 9.800.3 (Horizon Scraper)
+All available test cases:
+  SimpleAdaDeltaTestFunction
+      [AdaDeltaTest]
+...
+262 test cases
+```
+
 
 ## Release process
 

COPYRIGHT.txt

@@ -37,7 +37,9 @@ Copyright:
   Copyright 2018, Dan Timson
   Copyright 2019, Rahul Ganesh Prabhu
   Copyright 2019, Roberto Hueso <robertohueso96@gmail.com>
-  
+  Copyright 2019, Sayan Goswami <sayan.goswami.106@gmail.com>
+  Copyright 2020, Joe Dinius <josephwdinius@gmail.com>
+
 License: BSD-3-clause
   All rights reserved.
   .

HISTORY.md

@@ -1,12 +1,255 @@
-### ensmallen ?.??.?
-###### ????-??-??
-  * SGD callback test 32-bit safety (big number).
+### ensmallen 2.18.2: "Fairmount Bagel"
+###### 2022-02-13
+ * Update Catch2 to 2.13.8
+  ([#336](https://github.com/mlpack/ensmallen/pull/336)).
+
+ * Fix epoch timing output
+  ([#337](https://github.com/mlpack/ensmallen/pull/337)).
+
+### ensmallen 2.18.1: "Fairmount Bagel"
+###### 2021-11-19
+ * Accelerate SGD test time
+   ([#330](https://github.com/mlpack/ensmallen/pull/300)).
+
+ * Fix potential infinite loop in CMAES
+   ([#331](https://github.com/mlpack/ensmallen/pull/331)).
+
+ * Fix SCD partial gradient test
+   ([#332](https://github.com/mlpack/ensmallen/pull/332)).
+
+### ensmallen 2.18.0: "Fairmount Bagel"
+###### 2021-10-20
+ * Add gradient value clipping and gradient norm scaling callback
+   ([#315](https://github.com/mlpack/ensmallen/pull/315)).
+
+ * Remove superfluous CMake option to build the tests
+   ([#313](https://github.com/mlpack/ensmallen/pull/313)).
+
+ * Bump minimum Armadillo version to 9.800
+   ([#318](https://github.com/mlpack/ensmallen/pull/318)).
+
+ * Update Catch2 to 2.13.7
+   ([#322](https://github.com/mlpack/ensmallen/pull/322)).
+
+ * Remove redundant template argument for C++20 compatibility
+   ([#324](https://github.com/mlpack/ensmallen/pull/324)).
+
+ * Fix MOEAD test stability
+   ([#327](https://github.com/mlpack/ensmallen/pull/327)).
+
+### ensmallen 2.17.0: "Pachis Din Me Pesa Double"
+###### 2021-07-06
+ * CheckArbitraryFunctionTypeAPI extended for MOO support
+   ([#283](https://github.com/mlpack/ensmallen/pull/283)).
+
+ * Refactor NSGA2
+   ([#263](https://github.com/mlpack/ensmallen/pull/263),
+   [#304](https://github.com/mlpack/ensmallen/pull/304)).
+
+ * Add Indicators for Multiobjective optimizers
+   ([#285](https://github.com/mlpack/ensmallen/pull/285)).
+
+ * Make Callback flexible for MultiObjective Optimizers
+   ([#289](https://github.com/mlpack/ensmallen/pull/289)).
+
+ * Add ZDT Test Suite
+   ([#273](https://github.com/mlpack/ensmallen/pull/273)).
+
+ * Add MOEA-D/DE Optimizer
+   ([#269](https://github.com/mlpack/ensmallen/pull/269)).
+
+ * Introduce Policy Methods for MOEA/D-DE
+   ([#293](https://github.com/mlpack/ensmallen/pull/293)).
+
+ * Add Das-Dennis weight initialization method
+   ([#295](https://github.com/mlpack/ensmallen/pull/295)).
+
+ * Add Dirichlet Weight Initialization
+   ([#296](https://github.com/mlpack/ensmallen/pull/296)).
+
+ * Improved installation and compilation instructions
+   ([#300](https://github.com/mlpack/ensmallen/pull/300)).
+
+ * Disable building the tests by default for faster installation
+   ([#303](https://github.com/mlpack/ensmallen/pull/303)).
+
+ * Modify matrix initialisation to take into account
+   default element zeroing in Armadillo 10.5
+   ([#305](https://github.com/mlpack/ensmallen/pull/305)).
+
+### ensmallen 2.16.2: "Severely Dented Can Of Polyurethane"
+###### 2021-03-24
+ * Fix CNE test trials
+   ([#267](https://github.com/mlpack/ensmallen/pull/267)).
+
+ * Update Catch2 to 2.13.4
+   ([#268](https://github.com/mlpack/ensmallen/pull/268)).
+
+ * Fix typos in documentation
+   ([#270](https://github.com/mlpack/ensmallen/pull/270),
+    [#271](https://github.com/mlpack/ensmallen/pull/271)).
+
+ * Add clarifying comments in problems/ implementations
+   ([#276](https://github.com/mlpack/ensmallen/pull/276)).
+
+### ensmallen 2.16.1: "Severely Dented Can Of Polyurethane"
+###### 2021-03-02
+ * Fix test compilation issue when `ENS_USE_OPENMP` is set
+   ([#255](https://github.com/mlpack/ensmallen/pull/255)).
+
+ * Fix CNE initial population generation to use normal distribution
+   ([#258](https://github.com/mlpack/ensmallen/pull/258)).
+
+ * Fix compilation warnings
+   ([#259](https://github.com/mlpack/ensmallen/pull/259)).
+
+ * Remove `AdamSchafferFunctionN2Test` test from Adam test suite to prevent
+   spurious issue on some aarch64 ([#265](https://github.com/mlpack/ensmallen/pull/259)).
+
+### ensmallen 2.16.0: "Severely Dented Can Of Polyurethane"
+###### 2021-02-11
+ * Expand README with example installation and add simple example program
+   showing usage of the L-BFGS optimizer
+   ([#248](https://github.com/mlpack/ensmallen/pull/248)).
+
+ * Refactor tests to increase stability and reduce random errors
+   ([#249](https://github.com/mlpack/ensmallen/pull/249)).
+
+### ensmallen 2.15.1: "Why Can't I Manage To Grow Any Plants?"
+###### 2020-11-05
+ * Fix include order to ensure traits is loaded before reports
+   ([#239](https://github.com/mlpack/ensmallen/pull/239)).
+
+### ensmallen 2.15.0: "Why Can't I Manage To Grow Any Plants?"
+###### 2020-11-01
+ * Make a few tests more robust
+   ([#228](https://github.com/mlpack/ensmallen/pull/228)).
+
+ * Add release date to version information. ([#226](https://github.com/mlpack/ensmallen/pull/226))
+
+ * Fix typo in release script
+   ([#236](https://github.com/mlpack/ensmallen/pull/236)).
+
+### ensmallen 2.14.2: "No Direction Home"
+###### 2020-08-31
+ * Fix implementation of fonesca fleming problem function f1 and f2
+   type usage and negative signs. ([#223](https://github.com/mlpack/ensmallen/pull/223))
+
+### ensmallen 2.14.1: "No Direction Home"
+###### 2020-08-19
+ * Fix release script (remove hardcoded information, trim leading whitespaces
+   introduced by `wc -l` in MacOS)
+    ([#216](https://github.com/mlpack/ensmallen/pull/216),
+     [#220](https://github.com/mlpack/ensmallen/pull/220)).
+
+ * Adjust tolerance for AugLagrangian convergence based on element type
+   ([#217](https://github.com/mlpack/ensmallen/pull/217)).
+
+### ensmallen 2.14.0: "No Direction Home"
+###### 2020-08-10
+ * Add NSGA2 optimizer for multi-objective functions
+    ([#149](https://github.com/mlpack/ensmallen/pull/149)).
+
+ * Update automatic website update release script
+   ([#207](https://github.com/mlpack/ensmallen/pull/207)).
+
+ * Clarify and fix documentation for constrained optimizers
+   ([#201](https://github.com/mlpack/ensmallen/pull/201)).
+
+ * Fix L-BFGS convergence when starting from a minimum
+   ([#201](https://github.com/mlpack/ensmallen/pull/201)).
+
+* Add optimizer summary report callback
+   ([#213](https://github.com/mlpack/ensmallen/pull/213)).
+
+### ensmallen 2.13.0: "Automatically Automated Automation"
+###### 2020-07-15
+ * Fix CMake package export
+    ([#198](https://github.com/mlpack/ensmallen/pull/198)).
+
+ * Allow early stop callback to accept a lambda function
+   ([#165](https://github.com/mlpack/ensmallen/pull/165)).
+
+### ensmallen 2.12.1: "Stir Crazy"
+###### 2020-04-20
+ * Fix total number of epochs and time estimation for ProgressBar callback
+    ([#181](https://github.com/mlpack/ensmallen/pull/181)).
+
+ * Handle SpSubview_col and SpSubview_row in Armadillo 9.870
+    ([#194](https://github.com/mlpack/ensmallen/pull/194)).
+
+ * Minor documentation fixes
+    ([#197](https://github.com/mlpack/ensmallen/pull/197)).
+
+### ensmallen 2.12.0: "Stir Crazy"
+###### 2020-03-28
+ * Correction in the formulation of sigma in CMA-ES
+    ([#183](https://github.com/mlpack/ensmallen/pull/183)).
+
+ * Remove deprecated methods from PrimalDualSolver implementation
+    ([#185](https://github.com/mlpack/ensmallen/pull/185).
+
+ * Update logo ([#186](https://github.com/mlpack/ensmallen/pull/186)).
+
+### ensmallen 2.11.5: "The Poster Session Is Full"
+###### 2020-03-11
+  * Change "mathematical optimization" term to "numerical optimization" in the
+    documentation ([#177](https://github.com/mlpack/ensmallen/pull/177)).
+
+### ensmallen 2.11.4: "The Poster Session Is Full"
+###### 2020-03-03
+  * Require new HISTORY.md entry for each PR.
+    ([#171](https://github.com/mlpack/ensmallen/pull/171),
+     [#172](https://github.com/mlpack/ensmallen/pull/172),
+     [#175](https://github.com/mlpack/ensmallen/pull/175)).
+
+  * Update/fix example documentation
+    ([#174](https://github.com/mlpack/ensmallen/pull/174)).
+
+### ensmallen 2.11.3: "The Poster Session Is Full"
+###### 2020-02-19
+  * Prevent spurious compiler warnings
+    ([#161](https://github.com/mlpack/ensmallen/pull/161)).
+
+  * Fix minor memory leaks
+    ([#167](https://github.com/mlpack/ensmallen/pull/167)).
+
+  * Revamp CMake configuration
+    ([#152](https://github.com/mlpack/ensmallen/pull/152)).
+
+### ensmallen 2.11.2: "The Poster Session Is Full"
+###### 2020-01-16
+  * Allow callback instantiation for SGD based optimizer
+    ([#138](https://github.com/mlpack/ensmallen/pull/155)).
+
+  * Minor test stability fixes on i386
+    ([#156](https://github.com/mlpack/ensmallen/pull/156)).
+
+  * Fix Lookahead MaxIterations() check.
+    ([#159](https://github.com/mlpack/ensmallen/pull/159)).
+
+### ensmallen 2.11.1: "The Poster Session Is Full"
+###### 2019-12-28
+  * Fix Lookahead Synchronization period type
+    ([#153](https://github.com/mlpack/ensmallen/pull/153)).
+
+### ensmallen 2.11.0: "The Poster Session Is Full"
+###### 2019-12-24
+  * Add Lookahead
+    ([#138](https://github.com/mlpack/ensmallen/pull/138)).
+
+  * Add AdaBound and AMSBound
+    ([#137](https://github.com/mlpack/ensmallen/pull/137)).
+
+### ensmallen 2.10.5: "Fried Chicken"
+###### 2019-12-13
+  * SGD callback test 32-bit safety (big number)
     ([#143](https://github.com/mlpack/ensmallen/pull/143)).
 
   * Use "arbitrary" and "separable" terms in static function type checks
     ([#145](https://github.com/mlpack/ensmallen/pull/145)).
 
-  * Remove 'using namespace std' from problems/ files.
+  * Remove 'using namespace std' from `problems/` files
     ([#147](https://github.com/mlpack/ensmallen/pull/147)).
 
 ### ensmallen 2.10.4: "Fried Chicken"

README.md

@@ -1,14 +1,17 @@
-**ensmallen** is a C++ header-only library for mathematical optimization.
+<h2 align="center">
+  <a href="http://ensmallen.org/"><img src="http://ensmallen.org/img/ensmallen_text.svg" style="background-color:rgba(0,0,0,0);" height=230 alt="ensmallen: a C++ header-only library for numerical optimization"></a>
+</h2>
+
+**ensmallen** is a high-quality C++ library for non-linear numerical optimization.
+
+ensmallen provides many types of optimizers that can be used
+for virtually any numerical optimization task.
+This includes gradient descent techniques, gradient-free optimizers,
+and constrained optimization.
+ensmallen also allows optional callbacks to customize the optimization process.
 
 Documentation and downloads: http://ensmallen.org
 
-ensmallen provides a simple set of abstractions for writing an objective
-function to optimize. It also provides a large set of standard and cutting-edge
-optimizers that can be used for virtually any mathematical optimization task.
-These include full-batch gradient descent techniques, small-batch techniques,
-gradient-free optimizers, and constrained optimization.
-
-
 ### Requirements
 
 * C++ compiler with C++11 support
@@ -16,12 +19,82 @@ gradient-free optimizers, and constrained optimization.
 * OpenBLAS or Intel MKL or LAPACK (see Armadillo site for details)
 
 
+### Installation
+
+ensmallen can be installed in several ways: either manually or via cmake, 
+with or without root access.
+
+The cmake based installation will check the requirements 
+and optionally build the tests. If cmake 3.3 (or a later version) 
+is not already available on your system, it can be obtained 
+from [cmake.org](https://cmake.org). If you are using an older 
+system such as RHEL 7 or CentOS 7, an updated version of cmake 
+is also available via the EPEL repository (see the `cmake3` package).
+
+Example cmake based installation with root access:
+
+```
+mkdir build
+cd build
+cmake ..
+sudo make install
+```
+
+Example cmake based installation without root access, 
+installing into `/home/blah/` (adapt as required):
+
+```
+mkdir build
+cd build
+cmake .. -DCMAKE_INSTALL_PREFIX:PATH=/home/blah/
+make install
+```
+
+The above will create a directory named `/home/blah/include/` 
+and place all ensmallen headers there.
+
+To optionally build and run the tests
+(after running cmake as above),
+use the following additional commands:
+
+```
+make ensmallen_tests
+./ensmallen_tests --durations yes
+```
+
+Manual installation involves simply copying the `include/ensmallen.hpp` header 
+***and*** the associated `include/ensmallen_bits` directory to a location 
+such as `/usr/include/` which is searched by your C++ compiler.
+If you can't use `sudo` or don't have write access to `/usr/include/`, 
+use a directory within your own home directory (eg. `/home/blah/include/`).
+
+
+### Example Compilation
+
+If you have installed ensmallen in a standard location such as `/usr/include/`:
+
+    g++ prog.cpp -o prog -O2 -larmadillo
+    
+If you have installed ensmallen in a non-standard location, 
+such as `/home/blah/include/`, you will need to make sure 
+that your C++ compiler searches `/home/blah/include/` 
+by explicitly specifying the directory as an argument/option. 
+For example, using the `-I` switch in gcc and clang:
+
+    g++ prog.cpp -o prog -O2 -I /home/blah/include/ -larmadillo
+
+
+### Example Optimization
+
+See [`example.cpp`](example.cpp) for example usage of the L-BFGS optimizer 
+in a linear regression setting.
+
+
 ### License
 
-Unless stated otherwise, the source code for **ensmallen**
-is licensed under the 3-clause BSD license (the "License").
-A copy of the License is included in the "LICENSE.txt" file.
-You may also obtain a copy of the License at
+Unless stated otherwise, the source code for **ensmallen** is licensed under the
+3-clause BSD license (the "License").  A copy of the License is included in the
+"LICENSE.txt" file.  You may also obtain a copy of the License at
 http://opensource.org/licenses/BSD-3-Clause
 
 
@@ -31,10 +104,22 @@ Please cite the following paper if you use ensmallen in your research and/or
 software. Citations are useful for the continued development and maintenance of
 the library.
 
-* S. Bhardwaj, R. Curtin, M. Edel, Y. Mentekidis, C. Sanderson.  
-  [ensmallen: a flexible C++ library for efficient function optimization](http://www.ensmallen.org/files/ensmallen_2018.pdf).  
-  Workshop on Systems for ML and Open Source Software at NIPS 2018.
+* Ryan R. Curtin, Marcus Edel, Rahul Ganesh Prabhu, Suryoday Basak, Zhihao Lou, Conrad Sanderson.  
+  [The ensmallen library for flexible numerical optimization](https://jmlr.org/papers/volume22/20-416/20-416.pdf).  
+  Journal of Machine Learning Research, Vol. 22, No. 166, 2021.
 
+```
+@article{ensmallen_JMLR_2021,
+  author  = {Ryan R. Curtin and Marcus Edel and Rahul Ganesh Prabhu and Suryoday Basak and Zhihao Lou and Conrad Sanderson},
+  title   = {The ensmallen library for flexible numerical optimization},
+  journal = {Journal of Machine Learning Research},
+  year    = {2021},
+  volume  = {22},
+  number  = {166},
+  pages   = {1--6},
+  url     = {http://jmlr.org/papers/v22/20-416.html}
+}
+```
 
 ### Developers and Contributors
 
@@ -64,3 +149,4 @@ the library.
 * Dan Timson
 * N Rajiv Vaidyanathan
 * Roberto Hueso
+* Sayan Goswami

doc/callbacks.md

@@ -23,7 +23,7 @@ MomentumSGD optimizer(0.01, 32, 100000, 1e-5, true, MomentumUpdate(0.5));
 optimizer.Optimize(f, coordinates, PrintLoss());
 ```
 
-</details>  
+</details>
 
 Passing multiple callbacks is just the same as passing a single callback:
 
@@ -42,7 +42,7 @@ MomentumSGD optimizer(0.01, 32, 100000, 1e-5, true, MomentumUpdate(0.5));
 optimizer.Optimize(f, coordinates, PrintLoss(), EarlyStopAtMinLoss());
 ```
 
-</details>  
+</details>
 
 It is also possible to pass a callback instantiation that allows accessing of
 internal callback parameters at a later state:
@@ -84,12 +84,20 @@ has been made.
 
  * `EarlyStopAtMinLoss()`
  * `EarlyStopAtMinLoss(`_`patience`_`)`
+ * `EarlyStopAtMinLoss(`_`func`_`)`
+ * `EarlyStopAtMinLoss(`_`func`_`,`_`patience`_`)`
 
 #### Attributes
 
 | **type** | **name** | **description** | **default** |
 |----------|----------|-----------------|-------------|
 | `size_t` | **`patience`** | The number of epochs to wait after the minimum loss has been reached. | `10` |
+| `std::function<double(const arma::mat&)>` | **`func`** | A callback to return immediate loss evaluated by the function. | |
+
+Note that for the `func` argument above, if a
+[different matrix type](#alternate-matrix-types) is desired, instead of using
+the class `EarlyStopAtMinLoss`, the class `EarlyStopAtMinLossType<MatType>`
+should be used.
 
 #### Examples:
 
@@ -104,9 +112,121 @@ RosenbrockFunction f;
 arma::mat coordinates = f.GetInitialPoint();
 optimizer.Optimize(f, coordinates, EarlyStopAtMinLoss());
 ```
+Another example of using lambda in the constructor.
+
+```c++
+// Generate random training data and labels.
+arma::mat trainingData(5, 100, arma::fill::randu);
+arma::Row<size_t> trainingLabels =
+    arma::randi<arma::Row<size_t>>(100, arma::distr_param(0, 1));
+// Generate a validation set.
+arma::mat validationData(5, 100, arma::fill::randu);
+arma::Row<size_t> validationLabels =
+    arma::randi<arma::Row<size_t>>(100, arma::distr_param(0, 1));
+
+// Create a LogisticRegressionFunction for both the training and validation data.
+LogisticRegressionFunction lrfTrain(trainingData, trainingLabels);
+LogisticRegressionFunction lrfValidation(validationData, validationLabels);
+
+// Create a callback that will terminate when the validation loss starts to
+// increase.
+EarlyStopAtMinLoss cb(
+    [&](const arma::mat& coordinates)
+    {
+      // You could also, e.g., print the validation loss here to watch it converge.
+      return lrfValidation.Evaluate(coordinates);
+    });
+
+arma::mat coordinates = lrfTrain.GetInitialPoint();
+SMORMS3 smorms3;
+smorms3.Optimize(lrfTrain, coordinates, cb);
+```
 
 </details>
 
+### GradClipByNorm
+
+One difficulty with optimization is that large parameter gradients can lead an
+optimizer to update the parameters strongly into a region where the loss
+function is much greater, effectively undoing much of the work done to get to
+the current solution. Such large updates during the optimization can cause a
+numerical overflow or underflow, often referred to as "exploding gradients". The
+exploding gradient problem can be caused by: Choosing the wrong learning rate
+which leads to huge updates in the gradients.  Failing to scale a data set
+leading to very large differences between data points.  Applying a loss function
+that computes very large error values.
+
+A common answer to the exploding gradients problem is to change the derivative
+of the error before applying the update step.  One option is to clip the norm
+`||g||` of the gradient `g` before a parameter update. So given the gradient,
+and a maximum norm value, the callback normalizes the gradient so that its
+L2-norm is less than or equal to the given maximum norm value.
+
+#### Constructors
+
+ * `GradClipByNorm(`_`maxNorm`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`maxNorm`** | The maximum clipping value. | |
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+optimizer.Optimize(f, coordinates, GradClipByNorm(0.3));
+```
+
+### GradClipByValue
+
+One difficulty with optimization is that large parameter gradients can lead an
+optimizer to update the parameters strongly into a region where the loss
+function is much greater, effectively undoing much of the work done to get to
+the current solution. Such large updates during the optimization can cause a
+numerical overflow or underflow, often referred to as "exploding gradients". The
+exploding gradient problem can be caused by: Choosing the wrong learning rate
+which leads to huge updates in the gradients.  Failing to scale a data set
+leading to very large differences between data points.  Applying a loss function
+that computes very large error values.
+
+A common answer to the exploding gradients problem is to change the derivative
+of the error before applying the update step.  One option is to clip the
+parameter gradient element-wise before a parameter update.
+
+#### Constructors
+
+ * `GradClipByValue(`_`min, max`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`min`** | The minimum value to clip to. | |
+| `double` | **`max`** | The maximum value to clip to. | |
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+optimizer.Optimize(f, coordinates, GradClipByValue(0, 1.3));
+```
+
 ### PrintLoss
 
 Callback that prints loss to stdout or a specified output stream.
@@ -171,6 +291,95 @@ optimizer.Optimize(f, coordinates, ProgressBar());
 
 </details>
 
+### Report
+
+Callback that prints a optimizer report to stdout or a specified output stream.
+
+#### Constructors
+
+ * `Report()`
+ * `Report(`_`iterationsPercentage`_`)`
+ * `Report(`_`iterationsPercentage, output`_`)`
+ * `Report(`_`iterationsPercentage, output, outputMatrixSize`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`iterationsPercentage`** | The number of iterations to report in percent, between [0, 1]. | `0.1` |
+| `std::ostream` | **`output`** | Ostream which receives output from this object. | `stdout` |
+| `size_t` | **`outputMatrixSize`** | The number of values to output for the function coordinates. | `4` |
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+optimizer.Optimize(f, coordinates, Report(0.1));
+```
+
+<details open>
+<summary>Click to collapse/expand example output.
+</summary>
+
+```
+Optimization Report
+--------------------------------------------------------------------------------
+
+Initial Coordinates:
+  -1.2000   1.0000
+
+Final coordinates:
+  -1.0490   1.1070
+
+iter          loss          loss change   |gradient|    step size     total time
+0             24.2          0             233           1             4.27e-05
+100           8.6           15.6          104           1             0.000215
+200           5.26          3.35          48.7          1             0.000373
+300           4.49          0.767         23.4          1             0.000533
+400           4.31          0.181         11.3          1             0.000689
+500           4.27          0.0431        5.4           1             0.000846
+600           4.26          0.012         2.86          1             0.00101
+700           4.25          0.00734       2.09          1             0.00117
+800           4.24          0.00971       1.95          1             0.00132
+900           4.22          0.0146        1.91          1             0.00148
+
+--------------------------------------------------------------------------------
+
+Version:
+ensmallen:                    2.13.0 (Automatically Automated Automation)
+armadillo:                    9.900.1 (Nocturnal Misbehaviour)
+
+Function:
+Number of functions:          1
+Coordinates rows:             2
+Coordinates columns:          1
+
+Loss:
+Initial                       24.2
+Final                         4.2
+Change                        20
+
+Optimizer:
+Maximum iterations:           1000
+Reached maximum iterations:   true
+Batchsize:                    1
+Iterations:                   1000
+Number of epochs:             1001
+Initial step size:            1
+Final step size:              1
+Coordinates max. norm:        233
+Evaluate calls:               1000
+Gradient calls:               1000
+Time (in seconds):            0.00163
+```
+
 ### StoreBestCoordinates
 
 Callback that stores the model parameter after every epoch if the objective
@@ -205,14 +414,14 @@ StoreBestCoordinates<arma::mat> cb;
 optimizer.Optimize(f, coordinates, cb);
 
 std::cout << "The optimized model found by AdaDelta has the "
-      << "parameters " << cb.BestCoordinatest();
+      << "parameters " << cb.BestCoordinates();
 ```
 
 </details>
 
 ## Callback States
 
-Callbacks are called at different states during the optimization process:
+Callbacks are called at several states during the optimization process:
 
 * At the beginning and end of the optimization process.
 * After any call to `Evaluate()` and `EvaluateConstraint`.
@@ -346,6 +555,23 @@ an estimate depending on `exactObjective` value.
 | `size_t` | **`epoch`** | The index of the current epoch. |
 | `double` | **`objective`** | Objective value of the current point. |
 
+### GenerationalStepTaken
+
+Called after the evolution of a single generation. Intended specifically for
+MultiObjective Optimizers.
+
+ * `GenerationalStepTaken(`_`optimizer, function, coordinates, objectives, frontIndices`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+| `ObjectivesVecType` | **`objectives`** | The set of calculated objectives so far. |
+| `IndicesType` | **`frontIndices`** | The indices of the members belonging to Pareto Front. |
+
 ## Custom Callbacks
 
 ### Learning rate scheduling

doc/function_types.md

@@ -30,6 +30,7 @@ The following optimizers can be used to optimize an arbitrary function:
  - [Simulated Annealing](#simulated-annealing-sa)
  - [CNE](#cne)
  - [DE](#de)
+ - [PSO](#pso)
  - [SPSA](#simultaneous-perturbation-stochastic-approximation-spsa)
 
 Each of these optimizers has an `Optimize()` function that is called as
@@ -551,16 +552,21 @@ and f'(x) compute some of the same intermediate quantities.
 Each of the implemented methods is allowed to have additional cv-modifiers
 (`static`, `const`, etc.).
 
-The following optimizers can be used with differentiable functions:
+The following optimizers can be used with differentiable separable functions:
 
+ - [AdaBound](#adabound)
  - [AdaDelta](#adadelta)
  - [AdaGrad](#adagrad)
  - [Adam](#adam)
  - [AdaMax](#adamax)
+ - [AMSBound](#amsbound)
  - [AMSGrad](#amsgrad)
  - [Big Batch SGD](#big-batch-sgd)
+ - [Eve](#eve)
+ - [FTML](#ftml-follow-the-moving-leader)
  - [IQN](#iqn)
  - [Katyusha](#katyusha)
+ - [Lookahead](#lookahead)
  - [Momentum SGD](#momentum-sgd)
  - [Nadam](#nadam)
  - [NadaMax](#nadamax)
@@ -574,8 +580,10 @@ The following optimizers can be used with differentiable functions:
  - [Stochastic Gradient Descent with Restarts (SGDR)](#stochastic-gradient-descent-with-restarts-sgdr)
  - [Snapshot SGDR](#snapshot-stochastic-gradient-descent-with-restarts)
  - [SMORMS3](#smorms3)
- - [SVRG](#standard-stochastic-variance-reduced-gradient-svrg)
  - [SPALeRA](#spalera-stochastic-gradient-descent-spalerasgd)
+ - [SWATS](#swats)
+ - [SVRG](#standard-stochastic-variance-reduced-gradient-svrg)
+ - [WNGrad](#wngrad)
 
 The example program below demonstrates the implementation and use of an
 arbitrary separable function.  The function used is the linear regression
@@ -833,23 +841,73 @@ int main()
 
 </details>
 
+
+## Multi-objective functions
+
+A multi-objective optimizer does not return just one set of coordinates at the
+minimum of all objective functions, but instead finds a *front* or *frontier* of
+possible coordinates that are Pareto-optimal (that is, no individual objective
+function's value can be reduced without increasing at least one other
+objective function).
+
+In order to optimize a multi-objective function with ensmallen, a `std::tuple<>`
+containing multiple `ArbitraryFunctionType`s ([see here](#arbitrary-functions))
+should be passed to a multi-objective optimizer's `Optimize()` function.
+
+An example below simultaneously optimizes the generalized Rosenbrock function
+in 6 dimensions and the Wood function using [NSGA2](#nsga2).
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+GeneralizedRosenbrockFunction rf(6);
+WoodFunction wf;
+std::tuple<GeneralizedRosenbrockFunction, WoodFunction> objectives(rf, wf);
+
+// Create an initial point (a random point in 6 dimensions).
+arma::mat coordinates(6, 1, arma::fill::randu);
+
+// `coordinates` will be set to the coordinates on the best front that minimize the
+// sum of objective functions, and `bestFrontSum` will be the sum of all objectives
+// at that coordinate set.
+NSGA2 nsga;
+double bestFrontSum = nsga.Optimize(objectives, coordinates);
+
+// Set `bestFront` to contain all of the coordinates on the best front.
+arma::cube bestFront = optimizer.ParetoFront();
+}
+```
+
+</details>
+
+*Note*: all multi-objective function optimizers have both the function `Optimize()` to find the
+best front, and also the function `ParetoFront()` to return all sets of solutions that are on the
+front.
+
+The following optimizers can be used with multi-objective functions:
+- [NSGA2](#nsga2)
+- [MOEA/D-DE](#moead)
+
 ## Constrained functions
 
 A constrained function is an objective function `f(x)` that is also subject to
 some constraints on `x`.  (For instance, perhaps a constraint could be that `x`
 is a positive semidefinite matrix.)  ensmallen is able to handle differentiable
 objective functions of this type---so, `f'(x)` must also be computable.  Given
-some set of constraints c_0(x), ... c_M(x), we can re-express our constrained
+some set of constraints `c_0(x)`, ..., `c_M(x)`, we can re-express our constrained
 objective function as
 
 ```
 f_C(x) = f(x) + c_0(x) + ... + c_M(x)
 ```
 
-where the constraint `c_i(x)` is `DBL_MAX` if it is not satisfied, and
-otherwise takes some real value.  For a "hard constraint", we can simply take
-`c_i(x) = 0` when it is satisfied.  But allowing `c_i(x)` to return anything
-allows us to handle "soft" constraints also.
+where the (soft) constraint `c_i(x)` is a positive value if it is not satisfied, and
+`0` if it is satisfied.  The soft constraint `c_i(x)` should take some value
+representing how far from a feasible solution `x` is.  It should be
+differentiable, since ensmallen's constrained optimizers will use the gradient
+of the constraint to find a feasible solution.
 
 In order to optimize a constrained function with ensmallen, a class
 implementing the API below is required.
@@ -872,16 +930,14 @@ class ConstrainedFunctionType
   size_t NumConstraints();
 
   // Evaluate constraint i at the parameters x.  If the constraint is
-  // unsatisfied, DBL_MAX should be returned.  If the constraint is satisfied,
-  // any real value can be returned.  The optimizer will add this value to its
-  // overall objective that it is trying to minimize.  (So, a hard constraint
-  // can just return 0 if it's satisfied.)
+  // unsatisfied, a value greater than 0 should be returned.  If the constraint
+  // is satisfied, 0 should be returned.  The optimizer will add this value to
+  // its overall objective that it is trying to minimize.
   double EvaluateConstraint(const size_t i, const arma::mat& x);
 
   // Evaluate the gradient of constraint i at the parameters x, storing the
-  // result in the given matrix g.  If this is a hard constraint you can set
-  // the gradient to 0.  If the constraint is not satisfied, it could be
-  // helpful to set the gradient in such a way that the gradient points in the
+  // result in the given matrix g.  If the constraint is not satisfied, the
+  // gradient should be set in such a way that the gradient points in the
   // direction where the constraint would be satisfied.
   void GradientConstraint(const size_t i, const arma::mat& x, arma::mat& g);
 };
@@ -1028,12 +1084,12 @@ int main()
   // use the PrimalDualSolver to solve it.
   // ens::PrimalDualSolver could be replaced with ens::LRSDP or other ensmallen
   // SDP solvers.
-  PrimalDualSolver<SDP<arma::sp_mat>> solver(sdp);
+  PrimalDualSolver solver;
   arma::mat X, Z;
   arma::vec ysparse, ydense;
   // ysparse, ydense, and Z hold the primal and dual variables found during the
   // optimization.
-  const double obj = solver.Optimize(X, ysparse, ydense, Z);
+  const double obj = solver.Optimize(sdp, X, ysparse, ydense, Z);
 
   std::cout << "SDP optimized with objective " << obj << "." << std::endl;
 }
@@ -1090,14 +1146,14 @@ class SquaredFunction
 
 int main()
 {
-  // The minimum is at x = [0 0 0].  Our initial point is chosen to be 
+  // The minimum is at x = [0 0 0].  Our initial point is chosen to be
   // [1.0, -1.0, 1.0].
   arma::fmat x("1.0 -1.0 1.0");
 
   // Create simulated annealing optimizer with default options.
   // The ens::SA<> type can be replaced with any suitable ensmallen optimizer
   // that is able to handle arbitrary functions.
-  ens::L_BFGS<> optimizer;
+  ens::L_BFGS optimizer;
   SquaredFunction f; // Create function to be optimized.
   optimizer.Optimize(f, x); // The optimizer will infer arma::fmat!
 

doc/optimizers.md

@@ -1,3 +1,65 @@
+## AdaBound
+
+*An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
+
+AdaBound is a variant of Adam which employs dynamic bounds on learning rates.
+
+#### Constructors
+
+ * `AdaBound()`
+ * `AdaBound(`_`stepSize, batchSize`_`)`
+ * `AdaBound(`_`stepSize, batchSize, finalLr, gamma, beta1, beta2, epsilon, maxIterations, tolerance, shuffle`_`)`
+ * `AdaBound(`_`stepSize, batchSize, finalLr, gamma, beta1, beta2, epsilon, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
+
+Note that the `AdaBound` class is based on the `AdaBoundType<`_`UpdateRule`_`>`
+class with _`UpdateRule`_` = AdaBoundUpdate`.
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`stepSize`** | Step size for each iteration. | `0.001` |
+| `size_t` | **`batchSize`** | Number of points to process in a single step. | `32` |
+| `double` | **`finalLr`** | The final (SGD) learning rate. | `0.1` |
+| `double` | **`gamma`** | The convergence speed of the bound functions. | `0.001` |
+| `double` | **`beta1`** | Exponential decay rate for the first moment estimates. | `0.9` |
+| `double` | **`beta2`** | Exponential decay rate for the weighted infinity norm estimates. | `0.999` |
+| `double` | **`epsilon`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
+| `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
+| `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
+
+The attributes of the optimizer may also be modified via the member methods
+`FinalLr()`, `Gamma()`, `StepSize()`, `BatchSize()`, `Beta1()`, `Beta2()`,
+`Eps()`, `MaxIterations()`, `Tolerance()`, `Shuffle()`, `ResetPolicy()`, and
+`ExactObjective()`.
+
+#### Examples
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+SphereFunction f(2);
+arma::mat coordinates = f.GetInitialPoint();
+
+AdaBound optimizer(0.001, 2, 0.1, 1e-3, 0.9, 0.999, 1e-8, 500000, 1e-3);
+optimizer.Optimize(f, coordinates);
+```
+
+</details>
+
+#### See also:
+
+ * [SGD in Wikipedia](https://en.wikipedia.org/wiki/Stochastic_gradient_descent)
+ * [SGD](#standard-sgd)
+ * [Adaptive Gradient Methods with Dynamic Bound of Learning Rate](https://arxiv.org/abs/1902.09843)
+ * [Adam: A Method for Stochastic Optimization](http://arxiv.org/abs/1412.6980)
+ * [Differentiable separable functions](#differentiable-separable-functions)
+
 ## AdaDelta
 
 *An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
@@ -23,7 +85,7 @@ gradients.
 | `double` | **`stepSize`** | Step size for each iteration. | `1.0` |
 | `size_t` | **`batchSize`**| Number of points to process in one step. | `32` |
 | `double` | **`rho`** | Smoothing constant. Corresponding to fraction of gradient to keep at each time step. | `0.95` |
-| `double` | **`epsilon`** | Value used to initialise the mean squared gradient parameter. | `1e-6` |
+| `double` | **`epsilon`** | Value used to initialize the mean squared gradient parameter. | `1e-6` |
 | `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
@@ -79,7 +141,7 @@ parameters.
 |----------|----------|-----------------|-------------|
 | `double` | **`stepSize`** | Step size for each iteration. | `0.01` |
 | `size_t` | **`batchSize`** | Number of points to process in one step. | `32` |
-| `double` | **`epsilon`** | Value used to initialise the mean squared gradient parameter. | `1e-8` |
+| `double` | **`epsilon`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
 | `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `tolerance` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
@@ -117,7 +179,7 @@ optimizer.Optimize(f, coordinates);
 
 *An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
 
-Adam is an an algorithm for first-order gradient-based optimization of
+Adam is an algorithm for first-order gradient-based optimization of
 stochastic objective functions, based on adaptive estimates of lower-order
 moments.
 
@@ -231,6 +293,68 @@ optimizer.Optimize(f, coordinates);
  * [Adam: A Method for Stochastic Optimization](http://arxiv.org/abs/1412.6980) (see section 7)
  * [Differentiable separable functions](#differentiable-separable-functions)
 
+## AMSBound
+
+*An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
+
+AMSBound is a variant of Adam which employs dynamic bounds on learning rates.
+
+#### Constructors
+
+ * `AMSBound()`
+ * `AMSBound(`_`stepSize, batchSize`_`)`
+ * `AMSBound(`_`stepSize, batchSize, finalLr, gamma, beta1, beta2, epsilon, maxIterations, tolerance, shuffle`_`)`
+ * `AMSBound(`_`stepSize, batchSize, finalLr, gamma, beta1, beta2, epsilon, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
+
+Note that the `AMSBound` class is based on the `AdaBoundType<`_`UpdateRule`_`>`
+class with _`UpdateRule`_` = AdaBoundUpdate`.
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`stepSize`** | Step size for each iteration. | `0.001` |
+| `size_t` | **`batchSize`** | Number of points to process in a single step. | `32` |
+| `double` | **`finalLr`** | The final (SGD) learning rate. | `0.1` |
+| `double` | **`gamma`** | The convergence speed of the bound functions. | `0.001` |
+| `double` | **`beta1`** | Exponential decay rate for the first moment estimates. | `0.9` |
+| `double` | **`beta2`** | Exponential decay rate for the weighted infinity norm estimates. | `0.999` |
+| `double` | **`epsilon`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
+| `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
+| `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
+
+The attributes of the optimizer may also be modified via the member methods
+`FinalLr()`, `Gamma()`, `StepSize()`, `BatchSize()`, `Beta1()`, `Beta2()`,
+`Eps()`, `MaxIterations()`, `Tolerance()`, `Shuffle()`, `ResetPolicy()`, and
+`ExactObjective()`.
+
+#### Examples
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+SphereFunction f(2);
+arma::mat coordinates = f.GetInitialPoint();
+
+AMSBound optimizer(0.001, 2, 0.1, 1e-3, 0.9, 0.999, 1e-8, 500000, 1e-3);
+optimizer.Optimize(f, coordinates);
+```
+
+</details>
+
+#### See also:
+
+ * [SGD in Wikipedia](https://en.wikipedia.org/wiki/Stochastic_gradient_descent)
+ * [SGD](#standard-sgd)
+ * [Adaptive Gradient Methods with Dynamic Bound of Learning Rate](https://arxiv.org/abs/1902.09843)
+ * [Adam: A Method for Stochastic Optimization](http://arxiv.org/abs/1412.6980)
+ * [Differentiable separable functions](#differentiable-separable-functions)
+
 ## AMSGrad
 
 *An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
@@ -300,19 +424,19 @@ optimizer uses [L-BFGS](#l-bfgs).
 
 #### Constructors
 
- * `AugLagrangian(`_`maxIterations, penaltyThresholdFactor sigmaUpdateFactor`_`)`
+ * `AugLagrangian(`_`maxIterations, penaltyThresholdFactor, sigmaUpdateFactor`_`)`
 
 #### Attributes
 
 | **type** | **name** | **description** | **default** |
 |----------|----------|-----------------|-------------|
 | `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `1000` |
-| `double` | **`penaltyThresholdFactor`** | When penalty threshold is updated, set it to this multiplied by the penalty. | `10.0` |
-| `double` | **`sigmaUpdateFactor`** | When sigma is updated, multiply it by this. | `0.25` |
+| `double` | **`penaltyThresholdFactor`** | When penalty threshold is updated, set it to this multiplied by the penalty. | `0.25` |
+| `double` | **`sigmaUpdateFactor`** | When sigma is updated, multiply it by this. | `10.0` |
 | `L_BFGS&` | **`lbfgs`** | Internal l-bfgs optimizer. | `L_BFGS()` |
 
 The attributes of the optimizer may also be modified via the member methods
-`MaxIterations()`, `PenaltyThresholdFactor()`, `SigmaUpdateFactor()` and `L_BFGS()`.
+`MaxIterations()`, `PenaltyThresholdFactor()`, `SigmaUpdateFactor()` and `LBFGS()`.
 
 <details open>
 <summary>Click to collapse/expand example code.
@@ -430,11 +554,11 @@ RosenbrockFunction f;
 arma::mat coordinates = f.GetInitialPoint();
 
 // Big-Batch SGD with the adaptive stepsize policy.
-BBS_BB optimizer(batchSize, 0.01, 0.1, 8000, 1e-4);
+BBS_BB optimizer(10, 0.01, 0.1, 8000, 1e-4);
 optimizer.Optimize(f, coordinates);
 
 // Big-Batch SGD with backtracking line search.
-BBS_Armijo optimizer2(batchSize, 0.01, 0.1, 8000, 1e-4);
+BBS_Armijo optimizer2(10, 0.01, 0.1, 8000, 1e-4);
 optimizer2.Optimize(f, coordinates);
 ```
 
@@ -505,11 +629,11 @@ RosenbrockFunction f;
 arma::mat coordinates = f.GetInitialPoint();
 
 // CMAES with the FullSelection policy.
-CMAES<> optimizer(0, -1, 1, 32, 200, 0.1e-4);
+CMAES<> optimizer(0, -1, 1, 32, 200, 1e-4);
 optimizer.Optimize(f, coordinates);
 
 // CMAES with the RandomSelection policy.
-ApproxCMAES<> approxOptimizer(batchSize, 0.01, 0.1, 8000, 1e-4);
+ApproxCMAES<> approxOptimizer(0, -1, 1. 32, 200, 1e-4);
 approxOptimizer.Optimize(f, coordinates);
 ```
 
@@ -642,7 +766,7 @@ Eve is a stochastic gradient based optimization method with locally and globally
 | `double` | **`beta2`** | Exponential decay rate for the weighted infinity norm estimates. | `0.999` |
 | `double` | **`beta3`** | Exponential decay rate for relative change. | `0.999` |
 | `double` | **`epsilon`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
-| `double` | **`clip`** | Clipping range to avoid extreme valus. | `10` |
+| `double` | **`clip`** | Clipping range to avoid extreme values. | `10` |
 | `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
@@ -1043,7 +1167,7 @@ proximalOptimizer.Optimize(f, coordinates);
 
 *An optimizer for [differentiable functions](#differentiable-functions)*
 
-L-BFGS is an optimization algorithm in the family of quasi-Newton methods that approximates the Broyden-Fletcher-Goldfarb-Shanno (BFGS) algorithm using a limited amount of computer memory.  
+L-BFGS is an optimization algorithm in the family of quasi-Newton methods that approximates the Broyden-Fletcher-Goldfarb-Shanno (BFGS) algorithm using a limited amount of computer memory.
 
 #### Constructors
 
@@ -1094,6 +1218,66 @@ optimizer.Optimize(f, coordinates);
  * [Limited-memory BFGS in Wikipedia](https://en.wikipedia.org/wiki/Limited-memory_BFGS)
  * [Differentiable functions](#differentiable-functions)
 
+## Lookahead
+
+*An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
+
+Lookahead is a stochastic gradient based optimization method which chooses a
+search direction by looking ahead at the sequence of "fast weights" generated
+by another optimizer.
+
+#### Constructors
+ * `Lookahead<>()`
+ * `Lookahead<>(`_`stepSize`_`)`
+ * `Lookahead<>(`_`stepSize, k`_`)`
+ * `Lookahead<>(`_`stepSize, k, maxIterations, tolerance, decayPolicy, exactObjective`_`)`
+ * `Lookahead<>(`_`baseOptimizer, stepSize, k, maxIterations, tolerance, decayPolicy, exactObjective`_`)`
+
+Note that `Lookahead<>` is based on the templated type
+`LookaheadType<`_`BaseOptimizerType, DecayPolicyType`_`>` with _`BaseOptimizerType`_` = Adam` and _`DecayPolicyType`_` = NoDecay`.
+
+Any optimizer that implements the differentiable separable functions interface
+can be paired with the `Lookahead` optimizer.
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `BaseOptimizerType` | **`baseOptimizer`** |  Optimizer for the forward step. | Adam |
+| `double` | **`stepSize`** | Step size for each iteration. | `0.5` |
+| `size_t` | **`k`** | The synchronization period. | `5` |
+| `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
+| `DecayPolicyType` | **`decayPolicy`** | Instantiated decay policy used to adjust the step size. | `DecayPolicyType()` |
+| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
+
+The attributes of the optimizer may also be modified via the member methods
+`BaseOptimizer()`, `StepSize()`, `K()`, `MaxIterations()`,
+`Tolerance()`, `DecayPolicy()` and `ExactObjective()`.
+
+#### Examples
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+
+Lookahead<> optimizer(0.5, 5, 100000, 1e-5);
+optimizer.Optimize(f, coordinates);
+```
+
+</details>
+
+#### See also:
+
+ * [SGD in Wikipedia](https://en.wikipedia.org/wiki/Stochastic_gradient_descent)
+ * [SGD](#standard-sgd)
+ * [Lookahead Optimizer: k steps forward, 1 step back](https://arxiv.org/abs/1907.08610)
+ * [Differentiable separable functions](#differentiable-separable-functions)
+
 ## LRSDP (low-rank SDP solver)
 
 *An optimizer for [semidefinite programs](#semidefinite-programs).*
@@ -1379,6 +1563,163 @@ optimizer.Optimize(f, coordinates);
  * [SGD in Wikipedia](https://en.wikipedia.org/wiki/Stochastic_gradient_descent)
  * [Differentiable separable functions](#differentiable-separable-functions)
 
+## MOEA/D-DE
+*An optimizer for arbitrary multi-objective functions.*
+MOEA/D-DE (Multi Objective Evolutionary Algorithm based on Decomposition - Differential Evolution) is a multi
+objective optimization algorithm. It works by decomposing the problem into a number of scalar optimization
+subproblems which are solved simultaneously per generation. MOEA/D in itself is a framework, this particular
+algorithm uses Differential Crossover followed by Polynomial Mutation to create offsprings which are then
+decomposed to form a Single Objective Problem. A diversity preserving mechanism is also employed which encourages
+a varied set of solution.
+
+#### Constructors
+* `MOEAD<`_`InitPolicyType, DecompPolicyType`_`>()`
+* `MOEAD<`_`InitPolicyType, DecompPolicyType`_`>(`_`populationSize, maxGenerations, crossoverProb,  neighborProb, neighborSize, distributionIndex, differentialWeight, maxReplace, epsilon, lowerBound, upperBound`_`)`
+
+The _`InitPolicyType`_ template parameter refers to the strategy used to
+initialize the reference directions.
+
+The following types are available:
+
+ * **`Uniform`**
+ * **`BayesianBootstrap`**
+ * **`Dirichlet`**
+
+The _`DecompPolicyType`_ template parameter refers to the strategy used to
+decompose the weight vectors to form a scalar objective function.
+
+The following types are available:
+
+ * **`Tchebycheff`**
+ * **`WeightedAverage`**
+ * **`PenaltyBoundaryIntersection`**
+
+For convenience the following types can be used:
+
+ * **`DefaultMOEAD`** (equivalent to `MOEAD<Uniform, Tchebycheff>`): utilizes Uniform method for weight initialization
+ and Tchebycheff for weight decomposition.
+ 
+ * **`BBSMOEAD`** (equivalent to `MOEAD<BayesianBootstrap, Tchebycheff>`): utilizes Bayesian Bootstrap method for weight initialization and Tchebycheff for weight decomposition.
+ 
+ * **`DirichletMOEAD`** (equivalent to `MOEAD<Dirichlet, Tchebycheff>`): utilizes Dirichlet sampling for weight init
+ and Tchebycheff for weight decomposition.
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `size_t` | **`populationSize`** | The number of candidates in the population. | `150` |
+| `size_t` | **`maxGenerations`** | The maximum number of generations allowed. | `300` |
+| `double` | **`crossoverProb`** | Probability that a crossover will occur. | `1.0` |
+| `double` | **`neighborProb`** | The probability of sampling from neighbor. | `0.9` |
+| `size_t` | **`neighborSize`** | The number of nearest-neighbours to consider per weight vector.  | `20` |
+| `double` | **`distributionIndex`** | The crowding degree of the mutation. | `20` |
+| `double` | **`differentialWeight`** | Amplification factor of the differentiation. | `0.5` |
+| `size_t` | **`maxReplace`** | The limit of solutions allowed to be replaced by a child. | `2`|
+| `double` | **`epsilon`** | Handles numerical stability after weight initialization. | `1E-10`|
+| `double`, `arma::vec` | **`lowerBound`** | Lower bound of the coordinates on the coordinates of the whole population during the search process. | `0` |
+| `double`, `arma::vec` | **`upperBound`** | Lower bound of the coordinates on the coordinates of the whole population during the search process. | `1` |
+| `InitPolicyType` | **`initPolicy`** | Instantiated init policy used to initialize weights. | `InitPolicyType()` |
+| `DecompPolicyType` | **`decompPolicy`** | Instantiated decomposition policy used to create scalar objective problem. | `DecompPolicyType()` |
+
+Attributes of the optimizer may also be changed via the member methods
+`PopulationSize()`, `MaxGenerations()`, `CrossoverRate()`, `NeighborProb()`, `NeighborSize()`, `DistributionIndex()`,
+`DifferentialWeight()`, `MaxReplace()`, `Epsilon()`, `LowerBound()`, `UpperBound()`, `InitPolicy()` and `DecompPolicy()`.
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+SchafferFunctionN1<arma::mat> SCH;
+arma::vec lowerBound("-10 -10");
+arma::vec upperBound("10 10");
+DefaultMOEAD opt(300, 300, 1.0, 0.9, 20, 20, 0.5, 2, 1E-10, lowerBound, upperBound);
+typedef decltype(SCH.objectiveA) ObjectiveTypeA;
+typedef decltype(SCH.objectiveB) ObjectiveTypeB;
+arma::mat coords = SCH.GetInitialPoint();
+std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = SCH.GetObjectives();
+// obj will contain the minimum sum of objectiveA and objectiveB found on the best front.
+double obj = opt.Optimize(objectives, coords);
+// Now obtain the best front.
+arma::cube bestFront = opt.ParetoFront();
+```
+</details>
+
+#### See also
+* [MOEA/D-DE Algorithm](https://ieeexplore.ieee.org/document/4633340)
+* [Multi-objective Functions in Wikipedia](https://en.wikipedia.org/wiki/Test_functions_for_optimization#Test_functions_for_multi-objective_optimization)
+* [Multi-objective functions](#multi-objective-functions)
+
+## NSGA2
+
+*An optimizer for arbitrary multi-objective functions.*
+
+NSGA2 (Non-dominated Sorting Genetic Algorithm - II) is a multi-objective
+optimization algorithm. The algorithm works by generating a candidate population
+from a fixed starting point. At each stage of optimization, a new population of
+children is generated. This new population along with its predecessor is sorted
+using non-domination as the metric. Following this, the population is further
+segregated into fronts. A new population is generated from these fronts having
+size equal to that of the starting population.
+
+#### Constructors
+
+ * `NSGA2()`
+ * `NSGA2(`_`populationSize, maxGenerations, crossoverProb, mutationProb, mutationStrength, epsilon, lowerBound, upperBound`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `size_t` | **`populationSize`** | The number of candidates in the population. This should be at least 4 in size and a multiple of 4. | `100` |
+| `size_t` | **`maxGenerations`** | The maximum number of generations allowed for NSGA2. | `2000` |
+| `double` | **`crossoverProb`** | Probability that a crossover will occur. | `0.6` |
+| `double` | **`mutationProb`** | Probability that a weight will get mutated. | `0.3` |
+| `double` | **`mutationStrength`** | The range of mutation noise to be added. This range is between 0 and mutationStrength. | `0.001` |
+| `double` | **`epsilon`** | The value used internally to evaluate approximate equality in crowding distance based sorting. | `1e-6` |
+| `double`, `arma::vec` | **`lowerBound`** | Lower bound of the coordinates on the coordinates of the whole population during the search process. | `0` |
+| `double`, `arma::vec` | **`upperBound`** | Lower bound of the coordinates on the coordinates of the whole population during the search process. | `1` |
+
+Note that the parameters `lowerBound` and `upperBound` are overloaded. Data types of `double` or `arma::mat` may be used. If they are initialized as single values of `double`, then the same value of the bound applies to all the axes, resulting in an initialization following a uniform distribution in a hypercube. If they are initialized as matrices of `arma::mat`, then the value of `lowerBound[i]` applies to axis `[i]`; similarly, for values in `upperBound`. This results in an initialization following a uniform distribution in a hyperrectangle within the specified bounds.
+
+Attributes of the optimizer may also be changed via the member methods
+`PopulationSize()`, `MaxGenerations()`, `CrossoverRate()`, `MutationProbability()`, `MutationStrength()`, `Epsilon()`, `LowerBound()` and `UpperBound()`.
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+SchafferFunctionN1<arma::mat> SCH;
+arma::vec lowerBound("-1000 -1000");
+arma::vec upperBound("1000 1000");
+NSGA2 opt(20, 5000, 0.5, 0.5, 1e-3, 1e-6, lowerBound, upperBound);
+
+typedef decltype(SCH.objectiveA) ObjectiveTypeA;
+typedef decltype(SCH.objectiveB) ObjectiveTypeB;
+
+arma::mat coords = SCH.GetInitialPoint();
+std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = SCH.GetObjectives();
+
+// obj will contain the minimum sum of objectiveA and objectiveB found on the best front.
+double obj = opt.Optimize(objectives, coords);
+// Now obtain the best front.
+arma::cube bestFront = opt.Front();
+```
+
+</details>
+
+#### See also:
+
+ * [NSGA-II Algorithm](https://www.iitk.ac.in/kangal/Deb_NSGA-II.pdf)
+ * [Multi-objective Functions in Wikipedia](https://en.wikipedia.org/wiki/Test_functions_for_optimization#Test_functions_for_multi-objective_optimization)
+  * [Multi-objective functions](#multi-objective-functions)
+
 ## OptimisticAdam
 
 *An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
@@ -1387,7 +1728,7 @@ OptimisticAdam is an optimizer which implements the Optimistic Adam algorithm
 which uses Optmistic Mirror Descent with the Adam Optimizer.  It addresses the
 problem of limit cycling while training GANs (generative adversarial networks).
 It uses OMD to achieve faster regret rates in solving the zero sum game of
-training a GAN. It consistently achieves a smaller KL divergnce with~ respect to
+training a GAN. It consistently achieves a smaller KL divergence with~ respect to
 the true underlying data distribution.  The implementation here can be used with
 any differentiable separable function, not just GAN training.
 
@@ -1730,7 +2071,7 @@ optimizer.Optimize(f, coordinates);
 
 QHAdam is an optimizer that uses quasi-hyperbolic descent with the Adam
 optimizer.  This replaces the moment estimators of Adam with quasi-hyperbolic
-terms, and different values of the `v1` and `v2` parameters are equivalent to
+terms, and various values of the `v1` and `v2` parameters are equivalent to
 the following other optimizers:
 
  * When `v1 = v2 = 1`, `QHAdam` is equivalent to `Adam`.
@@ -1808,7 +2149,7 @@ RMSProp utilizes the magnitude of recent gradients to normalize the gradients.
 | `double` | **`stepSize`** | Step size for each iteration. | `0.01` |
 | `size_t` | **`batchSize`** | Number of points to process in each step. | `32` |
 | `double` | **`alpha`** | Smoothing constant, similar to that used in AdaDelta and momentum methods. | `0.99` |
-| `double` | **`epsilon`** | Value used to initialise the mean squared gradient parameter. | `1e-8` |
+| `double` | **`epsilon`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
 | `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
@@ -1884,7 +2225,7 @@ shorter type `SA<>` may be used instead of the equivalent
 
 | **type** | **name** | **description** | **default** |
 |----------|----------|-----------------|-------------|
-| `CoolingScheduleType` | **`coolingSchedule`** | Instantiated cooling schedule (default ExponentialSchedule). | **n/a** |
+| `CoolingScheduleType` | **`coolingSchedule`** | Instantiated cooling schedule (default ExponentialSchedule). | **CoolingScheduleType()** |
 | `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 indicates no limit). | `1000000` |
 | `double` | **`initT`** | Initial temperature. | `10000.0` |
 | `size_t` | **`initMoves`** | Number of initial iterations without changing temperature. | `1000` |
@@ -2325,7 +2666,7 @@ optimizer.Optimize(f, coordinates);
 *An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
 
 SMORMS3 is a hybrid of RMSprop, which is trying to estimate a safe and optimal
-distance based on curvature or perhaps just normalizing the stepsize in the
+distance based on curvature or perhaps just normalizing the step-size in the
 parameter space.
 
 #### Constructors
@@ -2341,7 +2682,7 @@ parameter space.
 |----------|----------|-----------------|-------------|
 | `double` | **`stepSize`** | Step size for each iteration. | `0.001` |
 | `size_t` | **`batchSize`** | Number of points to process at each step. | `32` |
-| `double` | **`epsilon`** | Value used to initialise the mean squared gradient parameter. | `1e-16` |
+| `double` | **`epsilon`** | Value used to initialize the mean squared gradient parameter. | `1e-16` |
 | `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the mini-batch order is shuffled; otherwise, each mini-batch is visited in linear order. | `true` |
@@ -2548,7 +2889,7 @@ the projection of Adam steps on the gradient subspace.
 | `size_t` | **`batchSize`** | Number of points to process at each step. | `32` |
 | `double` | **`beta1`** | Exponential decay rate for the first moment estimates. | `0.9` |
 | `double` | **`beta2`** | Exponential decay rate for the weighted infinity norm estimates. | `0.999` |
-| `double` | **`epsilon`** | Value used to initialise the mean squared gradient parameter. | `1e-16` |
+| `double` | **`epsilon`** | Value used to initialize the mean squared gradient parameter. | `1e-16` |
 | `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the mini-batch order is shuffled; otherwise, each mini-batch is visited in linear order. | `true` |

example.cpp

@@ -0,0 +1,70 @@
+// Example implementation of an objective function class for linear regression
+// and usage of the L-BFGS optimizer.
+// 
+// Compilation:
+// g++ example.cpp -o example -O3 -larmadillo
+
+
+#include <iostream>
+#include <armadillo>
+#include <ensmallen.hpp>
+
+
+class LinearRegressionFunction
+{
+  public:
+  
+  LinearRegressionFunction(arma::mat& X, arma::vec& y) : X(X), y(y) { }
+  
+  double EvaluateWithGradient(const arma::mat& theta, arma::mat& gradient)
+  {
+    const arma::vec tmp = X.t() * theta - y;
+    gradient = 2 * X * tmp;
+    return arma::dot(tmp,tmp);
+  }
+  
+  private:
+  
+  const arma::mat& X;
+  const arma::vec& y;
+};
+
+
+int main(int argc, char** argv)
+{
+  if (argc < 3)
+  {
+    std::cout << "usage: " << argv[0] << " n_dims n_points" << std::endl;
+    return -1;
+  }
+  
+  int n_dims   = atoi(argv[1]);
+  int n_points = atoi(argv[2]);
+  
+  // generate noisy dataset with a slight linear pattern
+  arma::mat X(n_dims, n_points, arma::fill::randu);
+  arma::vec y(        n_points, arma::fill::randu);
+  
+  for (size_t i = 0; i < n_points; ++i)
+  {
+    double a = arma::randu();
+    X(1, i) += a;
+    y(i)    += a;
+  }
+  
+  LinearRegressionFunction lrf(X, y);
+  
+  // create a Limited-memory BFGS optimizer object with default parameters
+  ens::L_BFGS opt;
+  opt.MaxIterations() = 10;
+  
+  // initial point (uniform random)
+  arma::vec theta(n_dims, arma::fill::randu);
+  
+  opt.Optimize(lrf, theta);
+  
+  // theta now contains the optimized parameters
+  theta.print("theta:");
+  
+  return 0;
+}

include/ensmallen.hpp

@@ -33,20 +33,24 @@
   #error "need Armadillo version 8.400 or later"
 #endif
 
-#include <cmath>
-#include <cstdlib>
-#include <cstdio>
-#include <cstring>
 #include <cctype>
-#include <climits>
 #include <cfloat>
+#include <climits>
+#include <cmath>
 #include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <map>
+#include <set>
+#include <limits>
+#include <sstream>
 #include <stdexcept>
+#include <string>
 #include <tuple>
 #include <utility>
-#include <iostream>
-#include <string>
-#include <sstream>
+#include <vector>
 
 // On Visual Studio, disable C4519 (default arguments for function templates)
 // since it's by default an error, which doesn't even make any sense because
@@ -61,17 +65,27 @@
 
 #include "ensmallen_bits/utility/any.hpp"
 #include "ensmallen_bits/utility/arma_traits.hpp"
+#include "ensmallen_bits/utility/indicators/epsilon.hpp"
+#include "ensmallen_bits/utility/indicators/igd_plus.hpp"
+
+// Contains traits, must be placed before report callback.
+#include "ensmallen_bits/function.hpp" // TODO: should move to function/
 
 // Callbacks.
 #include "ensmallen_bits/callbacks/callbacks.hpp"
 #include "ensmallen_bits/callbacks/early_stop_at_min_loss.hpp"
+#include "ensmallen_bits/callbacks/grad_clip_by_norm.hpp"
+#include "ensmallen_bits/callbacks/grad_clip_by_value.hpp"
 #include "ensmallen_bits/callbacks/print_loss.hpp"
 #include "ensmallen_bits/callbacks/progress_bar.hpp"
+#include "ensmallen_bits/callbacks/query_front.hpp"
+#include "ensmallen_bits/callbacks/report.hpp"
 #include "ensmallen_bits/callbacks/store_best_coordinates.hpp"
 #include "ensmallen_bits/callbacks/timer_stop.hpp"
 
 #include "ensmallen_bits/problems/problems.hpp" // TODO: should move to another place
 
+#include "ensmallen_bits/ada_bound/ada_bound.hpp"
 #include "ensmallen_bits/ada_delta/ada_delta.hpp"
 #include "ensmallen_bits/ada_grad/ada_grad.hpp"
 #include "ensmallen_bits/adam/adam.hpp"
@@ -84,14 +98,15 @@
 #include "ensmallen_bits/eve/eve.hpp"
 #include "ensmallen_bits/ftml/ftml.hpp"
 
-#include "ensmallen_bits/function.hpp" // TODO: should move to function/
-
 #include "ensmallen_bits/fw/frank_wolfe.hpp"
 #include "ensmallen_bits/gradient_descent/gradient_descent.hpp"
 #include "ensmallen_bits/grid_search/grid_search.hpp"
 #include "ensmallen_bits/iqn/iqn.hpp"
 #include "ensmallen_bits/katyusha/katyusha.hpp"
 #include "ensmallen_bits/lbfgs/lbfgs.hpp"
+#include "ensmallen_bits/lookahead/lookahead.hpp"
+#include "ensmallen_bits/moead/moead.hpp"
+#include "ensmallen_bits/nsga2/nsga2.hpp"
 #include "ensmallen_bits/padam/padam.hpp"
 #include "ensmallen_bits/parallel_sgd/parallel_sgd.hpp"
 #include "ensmallen_bits/pso/pso.hpp"

include/ensmallen_bits/ada_bound/ada_bound.hpp

@@ -0,0 +1,209 @@
+/**
+ * @file ada_bound.hpp
+ * @author Marcus Edel
+ *
+ * Class wrapper for the AdaBound and AMSBound update Policy.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_BOUND_HPP
+#define ENSMALLEN_ADA_BOUND_HPP
+
+#include <ensmallen_bits/sgd/sgd.hpp>
+#include "ada_bound_update.hpp"
+#include "ams_bound_update.hpp"
+
+namespace ens {
+
+/**
+ * AdaBound and AMSBound, employ dynamic bounds on learning rates to achieve a
+ * gradual and smooth transition from adaptive methods to SGD.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @inproceedings{Luo2019AdaBound,
+ *   author    = {Luo, Liangchen and Xiong, Yuanhao and Liu, Yan and Sun, Xu},
+ *   title     = {Adaptive Gradient Methods with Dynamic Bound of Learning
+ *                Rate},
+ *   booktitle = {Proceedings of the 7th International Conference on Learning
+ *                Representations},
+ *   month     = {May},
+ *   year      = {2019},
+ *   address   = {New Orleans, Louisiana}
+ * }
+ * @endcode
+ *
+ * AdaBound and AMSBound can optimize differentiable separable functions.
+ * For more details, see the documentation on function types included with this
+ * distribution or on the ensmallen website.
+ */
+template<typename UpdatePolicyType = AdaBoundUpdate,
+         typename DecayPolicyType = NoDecay>
+class AdaBoundType
+{
+ public:
+  /**
+   * Construct the AdaBoundType optimizer with the given function and
+   * parameters. AdaBoundType is sensitive to its parameters and hence a good
+   * hyper paramater selection is necessary as its default may not fit every
+   * case.
+   *
+   * The maximum number of iterations refers to the maximum number of
+   * points that are processed (i.e., one iteration equals one point; one
+   * iteration does not equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in a single step.
+   * @param finalLr The final (SGD) learning rate.
+   * @param gamma The convergence speed of the bound functions.
+   * @param beta1 Exponential decay rate for the first moment estimates.
+   * @param beta2 Exponential decay rate for the weighted infinity norm
+   *     estimates.
+   * @param epsilon Value used to initialise the mean squared gradient
+   *     parameter.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *     limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *     function is visited in linear order.
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *     call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
+   */
+  AdaBoundType(const double stepSize = 0.001,
+               const size_t batchSize = 32,
+               const double finalLr = 0.1,
+               const double gamma = 1e-3,
+               const double beta1 = 0.9,
+               const double beta2 = 0.999,
+               const double epsilon = 1e-8,
+               const size_t maxIterations = 100000,
+               const double tolerance = 1e-5,
+               const bool shuffle = true,
+               const bool resetPolicy = true,
+               const bool exactObjective = false);
+
+  /**
+   * Optimize the given function using AdaBoundType. The given starting point
+   * will be modified to store the finishing point of the algorithm, and the
+   * final objective value is returned. The DecomposableFunctionType is checked
+   * for API consistency at compile time.
+   *
+   * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(DecomposableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
+  {
+    return optimizer.template Optimize<DecomposableFunctionType, MatType,
+        GradType, CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(DecomposableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<DecomposableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the final (SGD) learning rate.
+  double FinalLr() const { return optimizer.UpdatePolicy().FinalLr(); }
+  //! Modify the final (SGD) learning rate.
+  double& FinalLr() { return optimizer.UpdatePolicy().FinalLr(); }
+
+  //! Get the convergence speed of the bound functions.
+  double Gamma() const { return optimizer.UpdatePolicy().Gamma(); }
+  //! Modify the convergence speed of the bound functions.
+  double& Gamma() { return optimizer.UpdatePolicy().Gamma(); }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return optimizer.UpdatePolicy().Beta1(); }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return optimizer.UpdatePolicy().Beta1(); }
+
+  //! Get the second moment coefficient.
+  double Beta2() const { return optimizer.UpdatePolicy().Beta2(); }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return optimizer.UpdatePolicy().Beta2(); }
+
+  //! Get the value used to initialise the mean squared gradient parameter.
+  double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
+  //! Modify the value used to initialise the mean squared gradient parameter.
+  double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
+  //! Get whether or not the update policy parameters are reset before
+  //! Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+ private:
+  //! The Stochastic Gradient Descent object with QHAdam policy.
+  SGD<UpdatePolicyType, DecayPolicyType> optimizer;
+};
+
+using AdaBound = AdaBoundType<AdaBoundUpdate>;
+using AMSBound = AdaBoundType<AMSBoundUpdate>;
+
+} // namespace ens
+
+// Include implementation.
+#include "ada_bound_impl.hpp"
+
+#endif

include/ensmallen_bits/ada_bound/ada_bound_impl.hpp

@@ -0,0 +1,47 @@
+/**
+ * @file qhadam_impl.hpp
+ * @author Niteya Shah
+ *
+ * Implementation of QHAdam class wrapper.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_BOUND_IMPL_HPP
+#define ENSMALLEN_ADA_BOUND_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "ada_bound.hpp"
+
+namespace ens {
+
+template<typename UpdatePolicyType, typename DecayPolicyType>
+AdaBoundType<UpdatePolicyType, DecayPolicyType>::AdaBoundType(
+    const double stepSize,
+    const size_t batchSize,
+    const double finalLr,
+    const double gamma,
+    const double beta1,
+    const double beta2,
+    const double epsilon,
+    const size_t maxIterations,
+    const double tolerance,
+    const bool shuffle,
+    const bool resetPolicy,
+    const bool exactObjective) :
+    optimizer(stepSize,
+              batchSize,
+              maxIterations,
+              tolerance,
+              shuffle,
+              UpdatePolicyType(finalLr, gamma, epsilon, beta1, beta2),
+              NoDecay(),
+              resetPolicy,
+              exactObjective)
+{ /* Nothing to do. */ }
+
+} // namespace ens
+
+ #endif

include/ensmallen_bits/ada_bound/ada_bound_update.hpp

@@ -0,0 +1,207 @@
+/**
+ * @file ada_bound_update.hpp
+ * @author Marcus Edel
+ *
+ * Implments the AdaBound Optimizer. AdaBound is a variant of Adam which
+ * employs dynamic bounds on learning rates.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_BOUND_UPDATE_HPP
+#define ENSMALLEN_ADA_BOUND_UPDATE_HPP
+
+namespace ens {
+
+/**
+ * AdaBound employs dynamic bounds on learning rates to achieve a gradual and
+ * smooth transition from adaptive methods to SGD.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @inproceedings{Luo2019AdaBound,
+ *   author    = {Luo, Liangchen and Xiong, Yuanhao and Liu, Yan and Sun, Xu},
+ *   title     = {Adaptive Gradient Methods with Dynamic Bound of Learning
+ *                Rate},
+ *   booktitle = {Proceedings of the 7th International Conference on Learning
+ *                Representations},
+ *   month     = {May},
+ *   year      = {2019},
+ *   address   = {New Orleans, Louisiana}
+ * }
+ * @endcode
+ */
+class AdaBoundUpdate
+{
+ public:
+  /**
+   * Construct the AdaBound update policy with the given parameters.
+   *
+   * @param finalLr The final (SGD) learning rate.
+   * @param gamma The convergence speed of the bound functions.
+   * @param epsilon The epsilon value used to initialise the squared gradient
+   *        parameter.
+   * @param beta1 The smoothing parameter.
+   * @param beta2 The second moment coefficient.
+   */
+  AdaBoundUpdate(const double finalLr = 0.1,
+                 const double gamma = 1e-3,
+                 const double epsilon = 1e-8,
+                 const double beta1 = 0.9,
+                 const double beta2 = 0.999) :
+    finalLr(finalLr),
+    gamma(gamma),
+    epsilon(epsilon),
+    beta1(beta1),
+    beta2(beta2),
+    iteration(0)
+  {
+    // Nothing to do.
+  }
+
+  //! Get the final (SGD) learning rate.
+  double FinalLr() const { return finalLr; }
+  //! Modify the final (SGD) learning rate.
+  double& FinalLr() { return finalLr; }
+
+  //! Get the convergence speed of the bound functions.
+  double Gamma() const { return finalLr; }
+  //! Modify the convergence speed of the bound functions.
+  double& Gamma() { return finalLr; }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return epsilon; }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return beta1; }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return beta1; }
+
+  //! Get the second moment coefficient.
+  double Beta2() const { return beta2; }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return beta2; }
+
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent AdaBoundUpdate object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AdaBoundUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent), first(true), initialStepSize(0)
+    {
+      m.zeros(rows, cols);
+      v.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for AdaBound.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Convenience typedefs.
+      typedef typename MatType::elem_type ElemType;
+
+      // Save the initial step size.
+      if (first)
+      {
+        first = false;
+        initialStepSize = stepSize;
+      }
+
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // Decay the first and second moment running average coefficient.
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      v *= parent.beta2;
+      v += (1 - parent.beta2) * (gradient % gradient);
+
+      const ElemType biasCorrection1 = 1.0 - std::pow(parent.beta1,
+          parent.iteration);
+      const ElemType biasCorrection2 = 1.0 - std::pow(parent.beta2,
+          parent.iteration);
+
+      const ElemType fl = parent.finalLr * stepSize / initialStepSize;
+      const ElemType lower = fl * (1.0 - 1.0 / (parent.gamma *
+          parent.iteration + 1));
+      const ElemType upper = fl * (1.0 + 1.0 / (parent.gamma *
+          parent.iteration));
+
+       // Applies bounds on actual learning rate.
+      iterate -= arma::clamp((stepSize *
+          std::sqrt(biasCorrection2) / biasCorrection1) / (arma::sqrt(v) +
+          parent.epsilon), lower, upper) % m;
+    }
+
+   private:
+    // Instantiated parent object.
+    AdaBoundUpdate& parent;
+
+    // The exponential moving average of gradient values.
+    GradType m;
+
+    // The exponential moving average of squared gradient values.
+    GradType v;
+
+    // Whether this is the first call of the Update method.
+    bool first;
+
+    // The initial (Adam) learning rate.
+    double initialStepSize;
+  };
+
+ private:
+  // The final (SGD) learning rate.
+  double finalLr;
+
+  // The convergence speed of the bound functions.
+  double gamma;
+
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+
+  // The smoothing parameter.
+  double beta1;
+
+  // The second moment coefficient.
+  double beta2;
+
+  // The number of iterations.
+  size_t iteration;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/ada_bound/ams_bound_update.hpp

@@ -0,0 +1,214 @@
+/**
+ * @file ams_bound_update.hpp
+ * @author Marcus Edel
+ *
+ * Implments the AMSBound Optimizer. AMSBound is a variant of Adam which
+ * employs dynamic bounds on learning rates.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_AMS_BOUND_UPDATE_HPP
+#define ENSMALLEN_AMS_BOUND_UPDATE_HPP
+
+namespace ens {
+
+/**
+ * AMSBound employs dynamic bounds on learning rates to achieve a gradual and
+ * smooth transition from adaptive methods to SGD.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @inproceedings{Luo2019AdaBound,
+ *   author    = {Luo, Liangchen and Xiong, Yuanhao and Liu, Yan and Sun, Xu},
+ *   title     = {Adaptive Gradient Methods with Dynamic Bound of Learning
+ *                Rate},
+ *   booktitle = {Proceedings of the 7th International Conference on Learning
+ *                Representations},
+ *   month     = {May},
+ *   year      = {2019},
+ *   address   = {New Orleans, Louisiana}
+ * }
+ * @endcode
+ */
+class AMSBoundUpdate
+{
+ public:
+  /**
+   * Construct the AMSBound update policy with the given parameters.
+   *
+   * @param finalLr The final (SGD) learning rate.
+   * @param gamma The convergence speed of the bound functions.
+   * @param epsilon The epsilon value used to initialise the squared gradient
+   *        parameter.
+   * @param beta1 The smoothing parameter.
+   * @param beta2 The second moment coefficient.
+   */
+  AMSBoundUpdate(const double finalLr = 0.1,
+                 const double gamma = 1e-3,
+                 const double epsilon = 1e-8,
+                 const double beta1 = 0.9,
+                 const double beta2 = 0.999) :
+      finalLr(finalLr),
+      gamma(gamma),
+      epsilon(epsilon),
+      beta1(beta1),
+      beta2(beta2),
+      iteration(0)
+  {
+    // Nothing to do.
+  }
+
+  //! Get the final (SGD) learning rate.
+  double FinalLr() const { return finalLr; }
+  //! Modify the final (SGD) learning rate.
+  double& FinalLr() { return finalLr; }
+
+  //! Get the convergence speed of the bound functions.
+  double Gamma() const { return finalLr; }
+  //! Modify the convergence speed of the bound functions.
+  double& Gamma() { return finalLr; }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return epsilon; }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return beta1; }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return beta1; }
+
+  //! Get the second moment coefficient.
+  double Beta2() const { return beta2; }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return beta2; }
+
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent AMSBoundUpdate object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AMSBoundUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent), first(true), initialStepSize(0)
+    {
+      m.zeros(rows, cols);
+      v.zeros(rows, cols);
+      vImproved.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for AMSBound.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Convenience typedefs.
+      typedef typename MatType::elem_type ElemType;
+
+      // Save the initial step size.
+      if (first)
+      {
+        first = false;
+        initialStepSize = stepSize;
+      }
+
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // Decay the first and second moment running average coefficient.
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      v *= parent.beta2;
+      v += (1 - parent.beta2) * (gradient % gradient);
+
+      const ElemType biasCorrection1 = 1.0 - std::pow(parent.beta1,
+          parent.iteration);
+      const ElemType biasCorrection2 = 1.0 - std::pow(parent.beta2,
+          parent.iteration);
+
+      const ElemType fl = parent.finalLr * stepSize / initialStepSize;
+      const ElemType lower = fl * (1.0 - 1.0 / (parent.gamma *
+          parent.iteration + 1));
+      const ElemType upper = fl * (1.0 + 1.0 / (parent.gamma *
+          parent.iteration));
+
+      // Element wise maximum of past and present squared gradients.
+      vImproved = arma::max(vImproved, v);
+
+      // Applies bounds on actual learning rate.
+      iterate -= arma::clamp((stepSize *
+          std::sqrt(biasCorrection2) / biasCorrection1) /
+          (arma::sqrt(vImproved) + parent.epsilon),  lower, upper) % m;
+    }
+
+   private:
+    // Instantiated parent object.
+    AMSBoundUpdate& parent;
+
+    // The exponential moving average of gradient values.
+    GradType m;
+
+    // The exponential moving average of squared gradient values.
+    GradType v;
+
+    // Whether this is the first call of the Update method.
+    bool first;
+
+    // The initial (Adam) learning rate.
+    double initialStepSize;
+
+    // The optimal squared gradient value.
+    GradType vImproved;
+  };
+
+ private:
+  // The final (SGD) learning rate.
+  double finalLr;
+
+  // The convergence speed of the bound functions.
+  double gamma;
+
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+
+  // The smoothing parameter.
+  double beta1;
+
+  // The second moment coefficient.
+  double beta2;
+
+  // The number of iterations.
+  size_t iteration;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/ada_delta/ada_delta.hpp

@@ -105,7 +105,8 @@ class AdaDelta
            CallbackTypes&&... callbacks)
   {
     return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
-        CallbackTypes...>(function, iterate, callbacks...);
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
   }
 
   //! Forward the MatType as GradType.

include/ensmallen_bits/ada_grad/ada_grad.hpp

@@ -101,7 +101,8 @@ class AdaGrad
            CallbackTypes&&... callbacks)
   {
     return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
-        CallbackTypes...>(function, iterate, callbacks...);
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
   }
 
   //! Forward the MatType as GradType.

include/ensmallen_bits/adam/adam.hpp

@@ -128,7 +128,7 @@ class AdamType
   {
     return optimizer.template Optimize<
         SeparableFunctionType, MatType, GradType, CallbackTypes...>(
-        function, iterate, callbacks...);
+        function, iterate, std::forward<CallbackTypes>(callbacks)...);
   }
 
   //! Forward the MatType as GradType.

include/ensmallen_bits/adam/optimisticadam_update.hpp

@@ -2,7 +2,7 @@
  * @file optimisticadam_update.hpp
  * @author Moksh Jain
  *
- * OptmisticAdam optimizer. Implements Optimistic Adam, an algorithm which 
+ * OptmisticAdam optimizer. Implements Optimistic Adam, an algorithm which
  * uses Optimistic Mirror Descent with the Adam optimizer.
  *
  * ensmallen is free software; you may redistribute it and/or modify it under
@@ -16,18 +16,18 @@
 namespace ens {
 
 /**
- * OptimisticAdam is an optimizer which implements the Optimistic Adam 
+ * OptimisticAdam is an optimizer which implements the Optimistic Adam
  * algorithm which uses Optmistic Mirror Descent with the Adam Optimizer.
  * It addresses the problem of limit cycling while training GANs. It uses
- * OMD to achieve faster regret rates in solving the zero sum game of 
- * training a GAN. It consistently achieves a smaller KL divergnce with 
+ * OMD to achieve faster regret rates in solving the zero sum game of
+ * training a GAN. It consistently achieves a smaller KL divergnce with
  * respect to the true underlying data distribution.
  *
  * For more information, see the following.
  *
  * @code
  * @article{
- *   author  = {Constantinos Daskalakis, Andrew Ilyas, Vasilis Syrgkanis, 
+ *   author  = {Constantinos Daskalakis, Andrew Ilyas, Vasilis Syrgkanis,
  *              Haoyang Zeng},
  *   title   = {Training GANs with Optimism},
  *   year    = {2017},

include/ensmallen_bits/aug_lagrangian/aug_lagrangian_impl.hpp

@@ -27,7 +27,8 @@ inline AugLagrangian::AugLagrangian(const size_t maxIterations,
     penaltyThresholdFactor(penaltyThresholdFactor),
     sigmaUpdateFactor(sigmaUpdateFactor),
     lbfgs(lbfgs),
-    terminate(false)
+    terminate(false),
+    sigma(0.0)
 {
 }
 
@@ -107,6 +108,10 @@ AugLagrangian::Optimize(
   // Track the last objective to compare for convergence.
   ElemType lastObjective = function.Evaluate(coordinates);
 
+  // Convergence tolerance---depends on the epsilon of the type we are using for
+  // optimization.
+  ElemType tolerance = 1e3 * std::numeric_limits<ElemType>::epsilon();
+
   // Then, calculate the current penalty.
   ElemType penalty = 0;
   for (size_t i = 0; i < function.NumConstraints(); i++)
@@ -134,6 +139,7 @@ AugLagrangian::Optimize(
     if (!lbfgs.Optimize(augfunc, coordinates, callbacks...))
       Info << "L-BFGS reported an error during optimization."
           << std::endl;
+    Info << "Done with L-BFGS: " << coordinates << "\n";
 
     const ElemType objective = function.Evaluate(coordinates);
 
@@ -142,7 +148,7 @@ AugLagrangian::Optimize(
 
     // Check if we are done with the entire optimization (the threshold we are
     // comparing with is arbitrary).
-    if (std::abs(lastObjective - objective) < 1e-10 &&
+    if (std::abs(lastObjective - objective) < tolerance &&
         augfunc.Sigma() > 500000)
     {
       lambda = std::move(augfunc.Lambda());
@@ -196,6 +202,13 @@ AugLagrangian::Optimize(
       // We multiply sigma by a constant value.
       augfunc.Sigma() *= sigmaUpdateFactor;
       Info << "Updated sigma to " << augfunc.Sigma() << "." << std::endl;
+      if (augfunc.Sigma() >= std::numeric_limits<ElemType>::max() / 2.0)
+      {
+        Warn << "AugLagrangian::Optimize(): sigma too large for element type; "
+            << "terminating." << std::endl;
+        Callback::EndOptimization(*this, function, coordinates, callbacks...);
+        return false;
+      }
     }
 
     terminate |= Callback::StepTaken(*this, function, coordinates,

include/ensmallen_bits/bigbatch_sgd/bigbatch_sgd.hpp

@@ -101,6 +101,12 @@ class BigBatchSGD
               const double tolerance = 1e-5,
               const bool shuffle = true,
               const bool exactObjective = false);
+
+  /**
+   * Clean any memory associated with the BigBatchSGD object.
+   */
+  ~BigBatchSGD();
+
   /**
    * Optimize the given function using big-batch SGD.  The given starting point
    * will be modified to store the finishing point of the algorithm, and the

include/ensmallen_bits/bigbatch_sgd/bigbatch_sgd_impl.hpp

@@ -38,6 +38,12 @@ BigBatchSGD<UpdatePolicyType>::BigBatchSGD(
     updatePolicy(UpdatePolicyType())
 { /* Nothing to do. */ }
 
+template<typename UpdatePolicyType>
+BigBatchSGD<UpdatePolicyType>::~BigBatchSGD()
+{
+  instUpdatePolicy.Clean();
+}
+
 //! Optimize the function (minimize).
 template<typename UpdatePolicyType>
 template<typename SeparableFunctionType,

include/ensmallen_bits/callbacks/callbacks.hpp

@@ -257,6 +257,7 @@ class Callback
   {
     // This will return immediately once a callback returns true.
     bool result = false;
+    (void)(objective);  // prevent spurious compiler warnings
     (void)std::initializer_list<bool>{ result =
         result || Callback::EvaluateFunction(callbacks, optimizer, function,
         coordinates, objective)... };
@@ -332,6 +333,8 @@ class Callback
   {
     // This will return immediately once a callback returns true.
     bool result = false;
+    (void)(constraint);  // prevent spurious compiler warnings
+    (void)(constraintValue);
     (void)std::initializer_list<bool>{ result =
         result || Callback::EvaluateConstraintFunction(callbacks, optimizer,
             function, coordinates, constraint, constraintValue)... };
@@ -478,6 +481,7 @@ class Callback
   {
     // This will return immediately once a callback returns true.
     bool result = false;
+    (void)(constraint);  // prevent spurious compiler warnings
     (void)std::initializer_list<bool>{ result =
         result || Callback::GradientConstraintFunction(callbacks, optimizer,
         function, coordinates, constraint, gradient)... };
@@ -509,6 +513,7 @@ class Callback
   {
     // This will return immediately once a callback returns true.
     bool result = false;
+    (void)(objective);  // prevent spurious compiler warnings
     (void)std::initializer_list<bool>{ result =
         result || Callback::EvaluateFunction(callbacks, optimizer, function,
         coordinates, objective)... };
@@ -584,6 +589,8 @@ class Callback
   {
     // This will return immediately once a callback returns true.
     bool result = false;
+    (void)(epoch);  // prevent spurious compiler warnings
+    (void)(objective);
     (void)std::initializer_list<bool>{ result =
         result || Callback::BeginEpochFunction(callbacks, optimizer, function,
         coordinates, epoch, objective)... };
@@ -672,6 +679,8 @@ class Callback
   {
     // This will return immediately once a callback returns true.
     bool result = false;
+    (void)(epoch);  // prevent spurious compiler warnings
+    (void)(objective);
     (void)std::initializer_list<bool>{ result =
         result || Callback::EndEpochFunction(callbacks, optimizer, function,
         coordinates, epoch, objective)... };
@@ -735,6 +744,79 @@ class Callback
                     MatType& /* coordinates */)
   { return false; }
 
+ /**
+  * Invoke the GenerationalStepTaken() callback if it exists.
+  * Specialization for MultiObjective case.
+  *
+  * @param callback The callback to call.
+  * @param optimizer The optimizer used to update the function.
+  * @param function Function to optimize.
+  * @param coordinates Starting point.
+  * @param objectives The set of calculated objectives so far.
+  * @param frontIndices The indices of the members belonging to Pareto Front.
+  */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename ObjectivesVecType,
+           typename IndicesType>
+  static typename std::enable_if<
+      callbacks::traits::HasGenerationalStepTakenSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, ObjectivesVecType,
+      IndicesType>::hasBool, bool>::type
+  GenerationalStepTakenFunction(CallbackType& callback,
+                                OptimizerType& optimizer,
+                                FunctionType& function,
+                                MatType& coordinates,
+                                ObjectivesVecType& objectives,
+                                IndicesType& frontIndices)
+  {
+    return const_cast<CallbackType&>(callback).GenerationalStepTaken(
+        optimizer, function, coordinates, objectives, frontIndices);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename ObjectivesVecType,
+           typename IndicesType>
+  static typename std::enable_if<
+      callbacks::traits::HasGenerationalStepTakenSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, ObjectivesVecType,
+      IndicesType>::hasVoid, bool>::type
+  GenerationalStepTakenFunction(CallbackType& callback,
+                                OptimizerType& optimizer,
+                                FunctionType& function,
+                                MatType& coordinates,
+                                ObjectivesVecType& objectives,
+                                IndicesType& frontIndices)
+  {
+    const_cast<CallbackType&>(callback).GenerationalStepTaken(
+        optimizer, function, coordinates, objectives, frontIndices);
+
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename ObjectivesVecType,
+           typename IndicesType>
+  static typename std::enable_if<
+      callbacks::traits::HasGenerationalStepTakenSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, ObjectivesVecType,
+      IndicesType>::hasNone, bool>::type
+  GenerationalStepTakenFunction(CallbackType& /* callback */,
+                                OptimizerType& /* optimizer */,
+                                FunctionType& /* function */,
+                                MatType& /* coordinates */,
+                                ObjectivesVecType& /* objectives */,
+                                IndicesType& /* frontIndices */)
+  { return false; }
+
   /**
    * Iterate over the callbacks and invoke the StepTaken() callback if it
    * exists.
@@ -760,8 +842,41 @@ class Callback
             function, coordinates)... };
      return result;
   }
-};
 
+/**
+ * Iterate over the callbacks and invoke the GenerationalStepTaken() callback if it
+ * exists.
+ *
+ * Specialization for MultiObjective case.
+ *
+ * @param optimizer The optimizer used to update the function.
+ * @param function Function to optimize.
+ * @param coordinates Starting point.
+ * @param objectives The set of calculated objectives so far.
+ * @param frontIndices The indices of the members belonging to Pareto Front.
+ * @param callbacks The callbacks container.
+ */
+template<typename OptimizerType,
+         typename FunctionType,
+         typename ObjectivesVecType,
+         typename IndicesType,
+         typename MatType,
+         typename ...CallbackTypes>
+  static bool GenerationalStepTaken(OptimizerType& optimizer,
+                                    FunctionType& functions,
+                                    MatType& coordinates,
+                                    ObjectivesVecType& objectives,
+                                    IndicesType& frontIndices,
+                                    CallbackTypes&... callbacks)
+{
+  // This will return immediately once a callback returns true.
+  bool result = false;
+  (void)std::initializer_list<bool>{ result =
+    result || Callback::GenerationalStepTakenFunction(callbacks, optimizer,
+      functions, coordinates, objectives, frontIndices)... };
+  return result;
+}
+};
 } // namespace ens
 
 #endif

include/ensmallen_bits/callbacks/early_stop_at_min_loss.hpp

@@ -1,6 +1,7 @@
 /**
  * @file early_stop_at_min_loss.hpp
  * @author Marcus Edel
+ * @author Omar Shrit
  *
  * Implementation of the early stop at minimum loss callback function.
  *
@@ -12,13 +13,16 @@
 #ifndef ENSMALLEN_CALLBACKS_EARLY_STOP_AT_MIN_LOSS_HPP
 #define ENSMALLEN_CALLBACKS_EARLY_STOP_AT_MIN_LOSS_HPP
 
+#include <functional>
+
 namespace ens {
 
 /**
  * Early stopping to terminate the optimization process early if the loss stops
  * decreasing.
  */
-class EarlyStopAtMinLoss
+template<typename MatType = arma::mat>
+class EarlyStopAtMinLossType
 {
  public:
   /**
@@ -28,12 +32,33 @@ class EarlyStopAtMinLoss
    * @param patienceIn The number of epochs to wait after the minimum loss has
    *    been reached or no improvement has been made (Default: 10).
    */
-  EarlyStopAtMinLoss(const size_t patienceIn = 10) :
-      patience(patienceIn),
+  EarlyStopAtMinLossType(const size_t patienceIn = 10) :
+      callbackUsed(false), 
+      patience(patienceIn), 
       bestObjective(std::numeric_limits<double>::max()),
       steps(0)
   { /* Nothing to do here */ }
 
+  /**
+   * Set up the early stop at min loss class, which keeps track of the minimum
+   * loss and stops the optimization process if the loss stops decreasing.
+   *
+   * @param func, callback to return immediate loss evaluated by the function
+   * @param patienceIn The number of epochs to wait after the minimum loss has
+   *    been reached or no improvement has been made (Default: 10).
+   */
+  EarlyStopAtMinLossType(
+      std::function<double(const MatType&)> func,
+      const size_t patienceIn = 10)
+    : callbackUsed(true), 
+      patience(patienceIn), 
+      bestObjective(std::numeric_limits<double>::max()),
+      steps(0), 
+      localFunc(func)
+  {
+    // Nothing to do here
+  }
+
   /**
    * Callback function called at the end of a pass over the data.
    *
@@ -43,13 +68,18 @@ class EarlyStopAtMinLoss
    * @param epoch The index of the current epoch.
    * @param objective Objective value of the current point.
    */
-  template<typename OptimizerType, typename FunctionType, typename MatType>
+  template<typename OptimizerType, typename FunctionType>
   bool EndEpoch(OptimizerType& /* optimizer */,
                 FunctionType& /* function */,
-                const MatType& /* coordinates */,
+                const MatType& coordinates,
                 const size_t /* epoch */,
-                const double objective)
+                double objective)
   {
+    if (callbackUsed)
+    {
+      objective = localFunc(coordinates);
+    } 
+
     if (objective < bestObjective)
     {
       steps = 0;
@@ -68,6 +98,9 @@ class EarlyStopAtMinLoss
   }
 
  private:
+  //! False if the first constructor is called, true if the user passed a lambda. 
+  bool callbackUsed;
+
   //! The number of epochs to wait before terminating the optimization process.
   size_t patience;
 
@@ -76,8 +109,19 @@ class EarlyStopAtMinLoss
 
   //! Locally-stored number of steps since the loss improved.
   size_t steps;
+
+  //! Function to call at the end of the epoch.
+  std::function<double(const MatType&)> localFunc;
 };
 
+/*
+ * Note that the using definition is temporary, this definition should
+ * be removed when releasing ensmallen 3.0
+ * The renaming of the class is only to avoid a major version bump
+ * because if the template type added to this class
+ */
+using EarlyStopAtMinLoss = EarlyStopAtMinLossType<arma::mat>;
+
 } // namespace ens
 
 #endif

include/ensmallen_bits/callbacks/grad_clip_by_norm.hpp

@@ -0,0 +1,60 @@
+/**
+ * @file grad_clip_by_norm.hpp
+ * @author Marcus Edel
+ *
+ * Clip the gradients by multiplying the unit vector of the gradients with the
+ * threshold.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_NORM_HPP
+#define ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_NORM_HPP
+
+namespace ens {
+
+/**
+ * Clip the gradients by multiplying the unit vector of the gradients with the
+ * threshold.
+ */
+class GradClipByNorm
+{
+ public:
+  /**
+   * Set up the gradient clip by norm callback class with the maximum clipping
+   * value.
+   *
+   * @param maxNorm The maximum clipping value.
+   */
+  GradClipByNorm(const double maxNorm) : maxNorm(maxNorm)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Callback function called at any call to Gradient().
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param gradient Matrix that holds the gradient.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void Gradient(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                MatType& gradient)
+  {
+    const double gradientNorm = arma::norm(gradient);
+    if (gradientNorm > maxNorm)
+      gradient = maxNorm * gradient / gradientNorm;
+  }
+
+ private:
+  //! The maximum clipping value for gradient clipping.
+  const double maxNorm;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/grad_clip_by_value.hpp

@@ -0,0 +1,60 @@
+/**
+ * @file grad_clip_by_value.hpp
+ * @author Marcus Edel
+ *
+ * Clips the gradient to a specified min and max.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_VALUE_HPP
+#define ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_VALUE_HPP
+
+namespace ens {
+
+/**
+ * Clip the gradient to a specified min and max.
+ */
+class GradClipByValue
+{
+ public:
+  /**
+   * Set up the gradient clip by value callback class with the min and max
+   * value.
+   *
+   * @param min The minimum value to clip to.
+   * @param max The maximum value to clip to.
+   */
+  GradClipByValue(const double min, const double max) : lower(min), upper(max)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Callback function called at any call to Gradient().
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param gradient Matrix that holds the gradient.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void Gradient(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                MatType& gradient)
+  {
+    gradient = arma::clamp(gradient, lower, upper);
+  }
+
+ private:
+  //! The minimum value to clip to.
+  const double lower;
+
+  //! The maximum value to clip to.
+  const double upper;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/progress_bar.hpp

@@ -12,6 +12,8 @@
 #ifndef ENSMALLEN_CALLBACKS_PROGRESS_BAR_HPP
 #define ENSMALLEN_CALLBACKS_PROGRESS_BAR_HPP
 
+#include <ensmallen_bits/function.hpp>
+
 namespace ens {
 
 /**
@@ -54,21 +56,21 @@ class ProgressBar
                          FunctionType& function,
                          MatType& /* coordinates */)
   {
-    static_assert(callbacks::traits::HasBatchSizeSignature<
+    static_assert(traits::HasBatchSizeSignature<
       OptimizerType>::value,
       "The OptimizerType does not have a correct definition of BatchSize(). "
       "Please check that the OptimizerType fully satisfies the requirements "
       "of the ProgressBar API; see the callbacks documentation for more "
       "details.");
 
-    static_assert(callbacks::traits::HasMaxIterationsSignature<
+    static_assert(traits::HasMaxIterationsSignature<
       OptimizerType>::value,
       "The OptimizerType does not have a correct definition of MaxIterations()."
       " Please check that the OptimizerType fully satisfies the requirements "
       "of the ProgressBar API; see the callbacks documentation for more "
       "details.");
 
-    static_assert(callbacks::traits::HasNumFunctionsSignature<
+    static_assert(traits::HasNumFunctionsSignature<
       FunctionType>::value,
       "The OptimizerType does not have a correct definition of NumFunctions(). "
       "Please check that the OptimizerType fully satisfies the requirements "
@@ -79,17 +81,9 @@ class ProgressBar
     if (function.NumFunctions() % optimizer.BatchSize() > 0)
       epochSize++;
 
-    if (!optimizer.MaxIterations())
-    {
-      Warn << "Maximum number of iterations not defined (no limit),"
-           << " no progress bar shown." << std::endl;
-    }
-    else
-    {
-      epochs = optimizer.MaxIterations() / epochSize;
-      if (optimizer.MaxIterations() % epochSize > 0)
-        epochs++;
-    }
+    epochs = optimizer.MaxIterations() / function.NumFunctions();
+    if (optimizer.MaxIterations() % function.NumFunctions() > 0)
+      epochs++;
 
     stepTimer.tic();
   }
@@ -136,8 +130,12 @@ class ProgressBar
   {
     if (newEpoch)
     {
-      output << "Epoch " << epoch << "/" << epochs << "\n";
-      output.flush();
+      output << "Epoch " << epoch;
+      if (epochs > 0)
+      {
+        output << "/" << epochs;
+      }
+      output << '\n';
       newEpoch = false;
     }
 
@@ -159,8 +157,8 @@ class ProgressBar
       }
     }
 
-    output << "] " << progress << "% - ETA: " << (size_t) stepTimer.toc() *
-        (epochSize - step + 1) % 60 << "s - loss: " <<
+    output << "] " << progress << "% - ETA: " << (size_t) (stepTimer.toc() *
+        (epochSize - step + 1)) % 60 << "s - loss: " <<
         objective / (double) step <<  "\r";
     output.flush();
 
@@ -218,10 +216,10 @@ class ProgressBar
         output << ".";
       }
     }
-
-    const size_t stepTime = epochTimer.toc() / (double) epochSize * 1000;
-    output << "] " << progress << "% - " << (size_t) epochTimer.toc() % 60
-        << "s " << stepTime << "ms/step " << "- loss: " << objective  <<  "\n";
+    const double epochTimerElapsed = epochTimer.toc();
+    const size_t stepTime = epochTimerElapsed / (double) epochSize * 1000;
+    output << "] " << progress << "% - " << epochTimerElapsed
+        << "s/epoch; " << stepTime << "ms/step; loss: " << objective  <<  "\n";
     output.flush();
   }
 

include/ensmallen_bits/callbacks/query_front.hpp

@@ -0,0 +1,85 @@
+/**
+ * @file query_front.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * Implementation of the query front callback function.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_QUERY_FRONT_HPP
+#define ENSMALLEN_CALLBACKS_QUERY_FRONT_HPP
+
+namespace ens {
+
+/**
+ * Query the current Pareto Front after every GenerationalStepTaken callback function.
+ */
+class QueryFront
+{
+ public:
+  /**
+   * Set up the query front callback class with the specified inputs.
+   *
+   * @param queryRate The frequency at which the Pareto Front is queried.
+   * @param paretoFrontArray A reference to a vector of cube to store the queried fronts.
+   */
+    QueryFront(const size_t queryRate, std::vector<arma::cube>& paretoFrontArray) :
+        queryRate(queryRate),
+        paretoFrontArray(paretoFrontArray),
+        genCounter(0)
+    { /* Nothing to do here */ }
+
+   /**
+    * Callback function called at the end of a single generational run.
+    *
+    * @param optimizer The optimizer used to update the function.
+    * @param function Function to optimize.
+    * @param coordinates Starting point.
+    * @param objectives The set of calculated objectives so far.
+    * @param frontIndices The indices of the members belonging to Pareto Front.
+    */
+    template<typename OptimizerType,
+             typename FunctionType,
+             typename MatType,
+             typename ObjectivesVecType,
+             typename IndicesType>
+    void GenerationalStepTaken(OptimizerType& opt,
+                               FunctionType& /* function */,
+                               const MatType& /* coordinates */,
+                               const ObjectivesVecType& objectives,
+                               const IndicesType& frontIndices)
+    {
+      arma::cube currentParetoFront{};
+
+      if (genCounter % queryRate == 0)
+      {
+        currentParetoFront.resize(objectives[0].n_rows, objectives[0].n_cols,
+            frontIndices[0].size());
+        for (size_t solutionIdx = 0; solutionIdx < frontIndices[0].size(); ++solutionIdx)
+        {
+          currentParetoFront.slice(solutionIdx) =
+              arma::conv_to<arma::mat>::from(objectives[frontIndices[0][solutionIdx]]);
+        }
+
+        paretoFrontArray.emplace_back(std::move(currentParetoFront));
+      }
+
+      ++genCounter;
+    }
+
+
+ private:
+  //! The rate of query.
+  size_t queryRate;
+  //! A reference to the array of pareto fronts.
+  std::vector<arma::cube>& paretoFrontArray;
+  //! A counter for the current generation.
+  size_t genCounter;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/report.hpp

@@ -0,0 +1,613 @@
+/**
+ * @file report.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of a simple report callback function.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_REPORT_HPP
+#define ENSMALLEN_CALLBACKS_REPORT_HPP
+
+#include <ensmallen_bits/function.hpp>
+#include <iomanip>
+
+namespace ens {
+
+/**
+ * A simple optimization report.
+ */
+class Report
+{
+ public:
+  /**
+   * Set up the report callback class with the given output stream.
+   *
+   * @param iterationsPercentageIn The number of iterations to report in
+   *     percent, between [0, 1]).
+   * @param outputIn Ostream which receives output from this object.
+   * @param outputMatrixSizeIn The number of values to output for the function
+   *     coordinates.
+   */
+  Report(const double iterationsPercentageIn = 0.1,
+         std::ostream& outputIn = arma::get_cout_stream(),
+         const size_t outputMatrixSizeIn = 4) :
+      iterationsPercentage(iterationsPercentageIn),
+      output(outputIn),
+      outputMatrixSize(outputMatrixSizeIn),
+      objective(0),
+      gradientNorm(0),
+      hasGradient(false),
+      hasEndEpoch(false),
+      gradientCalls(0),
+      evaluateCalls(0),
+      epochCalls(0)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Callback function called at the begin of the optimization process.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void BeginOptimization(OptimizerType& /* optimizer */,
+                         FunctionType& /* function */,
+                         MatType& coordinates)
+  {
+    initialCoordinates = coordinates;
+    optimizationTimer.tic();
+  }
+
+  /**
+   * Callback function called at the begin of the optimization process.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void EndOptimization(OptimizerType& optimizer,
+                       FunctionType& function,
+                       MatType& coordinates)
+  {
+    output << "Optimization Report" << std::endl;
+    output << std::string(80, '-') << std::endl << std::endl;
+
+    std::streamsize streamPrecision = output.precision(4);
+
+    if (coordinates.n_rows > outputMatrixSize ||
+        coordinates.n_cols > outputMatrixSize)
+    {
+      output << "Initial coordinates: " << std::endl;
+      TruncatePrint(initialCoordinates, outputMatrixSize);
+      output << std::endl << "Final coordinates: " << std::endl;
+      TruncatePrint(coordinates, outputMatrixSize);
+    }
+    else
+    {
+      output << "Initial Coordinates:" << std::endl << initialCoordinates.t();
+      output << std::endl << "Final coordinates:" << std::endl
+          << coordinates.t() << std::endl;
+    }
+
+    PrettyPrintElement("iter");
+    PrettyPrintElement("loss");
+    PrettyPrintElement("loss change");
+
+    if (hasGradient)
+      PrettyPrintElement("|gradient|");
+
+    if (!stepsizes.empty())
+      PrettyPrintElement("step size");
+
+    PrettyPrintElement("total time");
+    output << std::endl;
+
+    size_t iterationStep = objectives.size() / (iterationsPercentage * 100);
+    if (iterationStep <= 0)
+      iterationStep = 1;
+
+    for (size_t i = 0; i < objectives.size(); i += iterationStep)
+    {
+      PrettyPrintElement(i);
+      PrettyPrintElement(objectives[i]);
+      PrettyPrintElement(
+          i > 0 ? objectives[i - iterationStep] - objectives[i] : 0);
+
+      if (hasGradient)
+        PrettyPrintElement(gradientsNorm[i]);
+
+      if (!stepsizes.empty())
+        PrettyPrintElement(stepsizes[i]);
+
+      PrettyPrintElement(timings[i]);
+      output << std::endl;
+    }
+
+    output << std::endl << std::string(80, '-') << std::endl << std::endl;
+    output << "Version:" << std::endl;
+    PrettyPrintElement("ensmallen:", 30);
+    output << ens::version::as_string() << std::endl;
+    PrettyPrintElement("armadillo:", 30);
+    output << arma::arma_version::as_string() << std::endl << std::endl;
+
+    output << "Function:" << std::endl;
+    std::stringstream functionStream;
+
+    PrintNumFunctions(function, functionStream);
+    if (functionStream.rdbuf()->in_avail() > 0)
+      output << functionStream.str();
+
+    PrettyPrintElement("Coordinates rows:", 30);
+    output << coordinates.n_rows << std::endl;
+    PrettyPrintElement("Coordinates columns:", 30);
+    output << coordinates.n_cols << std::endl;
+    output << std::endl;
+
+    // If we did not take any steps, at least fill what the initial objective
+    // was.
+    const bool tookStep = (objectives.size() > 0);
+    if (objectives.size() == 0 && evaluateCalls > 0)
+    {
+      objectives.push_back(objective);
+      timings.push_back(optimizationTimer.toc());
+    }
+    else if (evaluateCalls == 0)
+    {
+      // It's not entirely clear how to compute the objective (since the
+      // function could implement many different ways of evaluating the
+      // objective), so issue an error and return.
+      output << "Objective never computed.  Did the optimization fail?"
+          << std::endl;
+      PrettyPrintElement("Time (in seconds):", 30);
+      output << optimizationTimer.toc() << std::endl;
+      return;
+    }
+
+    output << "Loss:" << std::endl;
+    PrettyPrintElement("Initial", 30);
+    output << objectives[0] << std::endl;
+    PrettyPrintElement("Final", 30);
+    output << objectives[objectives.size() - 1] << std::endl;
+    PrettyPrintElement("Change", 30);
+    output << objectives[0] - objectives[objectives.size() - 1] << std::endl;
+
+    output << std::endl << "Optimizer:" << std::endl;
+    std::stringstream optimizerStream;
+
+    PrintMaxIterations(optimizer, optimizerStream);
+    PrintBatchSize(optimizer, optimizerStream);
+    if (functionStream.rdbuf()->in_avail() > 0)
+      output << optimizerStream.str();
+
+    PrettyPrintElement("Iterations:", 30);
+    if (tookStep)
+      output << objectives.size() << std::endl;
+    else
+      output << "0 (No steps taken! Did the optimization fail?)" << std::endl;
+
+    if (epochCalls > 0)
+    {
+      PrettyPrintElement("Number of epochs:", 30);
+      output << epochCalls << std::endl;
+    }
+
+    if (!stepsizes.empty())
+    {
+      PrettyPrintElement("Initial step size:", 30);
+      output << stepsizes.front() << std::endl;
+
+      PrettyPrintElement("Final step size:", 30);
+      output << stepsizes.back() << std::endl;
+    }
+
+    if (hasGradient && gradientsNorm.size() > 0)
+    {
+      PrettyPrintElement("Coordinates max. norm:", 30);
+      output << *std::max_element(std::begin(gradientsNorm),
+          std::end(gradientsNorm)) << std::endl;
+    }
+
+    PrettyPrintElement("Evaluate calls:", 30);
+    output << evaluateCalls << std::endl;
+
+    if (hasGradient)
+    {
+      PrettyPrintElement("Gradient calls:", 30);
+      output << gradientCalls << std::endl;
+    }
+
+    PrettyPrintElement("Time (in seconds):", 30);
+    output << timings[timings.size() - 1] << std::endl;
+
+    // Restore precision.
+    output.precision(streamPrecision);
+  }
+
+  /**
+   * Callback function called at the beginning of a pass over the data.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void BeginEpoch(OptimizerType& /* optimizer */,
+                  FunctionType& /* function */,
+                  const MatType& /* coordinates */,
+                  const size_t /* epoch */,
+                  const double /* objective */)
+  {
+    epochCalls++;
+  }
+
+  /**
+   * Callback function called at the end of a pass over the data.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void EndEpoch(OptimizerType& optimizer,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                const size_t /* epoch */,
+                const double objective)
+  {
+    // In case StepTaken() has been called first we clear the existing data.
+    if (!hasEndEpoch)
+    {
+      hasEndEpoch = true;
+
+      objectives.clear();
+      timings.clear();
+      gradientsNorm.clear();
+      stepsizes.clear();
+    }
+
+    objectives.push_back(objective);
+    timings.push_back(optimizationTimer.toc());
+
+    if (hasGradient)
+      gradientsNorm.push_back(gradientNorm);
+
+    SaveStepSize(optimizer);
+  }
+
+  /**
+   * Callback function called once a step is taken.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void StepTaken(OptimizerType& optimizer,
+                 FunctionType& /* function */,
+                 const MatType& /* coordinates */)
+  {
+    if (!hasEndEpoch)
+    {
+      objectives.push_back(objective);
+      timings.push_back(optimizationTimer.toc());
+
+      if (hasGradient)
+        gradientsNorm.push_back(gradientNorm);
+
+      SaveStepSize(optimizer);
+    }
+  }
+
+  /**
+   * Callback function called at any call to Evaluate().
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param objectiveIn Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void Evaluate(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                const double objectiveIn)
+  {
+    objective = objectiveIn;
+    evaluateCalls++;
+  }
+
+  /**
+    * Callback function called at any call to EvaluateConstraint().
+    *
+    * @param optimizer The optimizer used to update the function.
+    * @param function Function to optimize.
+    * @param coordinates Starting point.
+    * @param constraint The index of the constraint;
+    * @param objectiveIn Objective value of the current point.
+    */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void EvaluateConstraint(OptimizerType& /* optimizer */,
+                          FunctionType& /* function */,
+                          const MatType& /* coordinates */,
+                          const size_t /* constraint */,
+                          const double objectiveIn)
+  {
+    objective += objectiveIn;
+    evaluateCalls++;
+  }
+
+  /**
+   * Callback function called at any call to Gradient().
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param gradientIn Matrix that holds the gradient.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void Gradient(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                const MatType& gradientIn)
+  {
+    hasGradient = true;
+    gradientNorm = arma::norm(gradientIn);
+    gradientCalls++;
+  }
+
+  /**
+   * Callback function called at any call to GradientConstraint().
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param constraint The index of the constraint;
+   * @param gradient Matrix that holds the gradient;
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void GradientConstraint(OptimizerType& optimizer,
+                          FunctionType& function,
+                          const MatType& coordinates,
+                          const size_t /* constraint */,
+                          const MatType& gradient)
+  {
+    Gradient(optimizer, function, coordinates, gradient);
+  }
+
+ private:
+  /**
+   * Helper function to print the number of function to the specified output
+   * stream.
+   *
+   * @param function The instantiated function that implements NumFunctions().
+   * @param stream The output stream.
+   */
+  template<typename FunctionType>
+  typename std::enable_if<
+      traits::HasNumFunctionsSignature<FunctionType>::value, void>::type
+  PrintNumFunctions(const FunctionType& function, std::stringstream& stream)
+  {
+    PrettyPrintElement(stream, "Number of functions:", 30);
+    stream << function.NumFunctions() << std::endl;
+  }
+
+  template<typename FunctionType>
+  typename std::enable_if<
+      !traits::HasNumFunctionsSignature<FunctionType>::value, void>::type
+  PrintNumFunctions(const FunctionType& /* function */,
+                    std::stringstream& /* stream */) { }
+
+  /**
+   * Helper function to output the max-iterations to the specified output
+   * stream.
+   *
+   * @param optimizer The instantiated optimizer that implements
+   *     MaxIterations().
+   * @param stream The output stream.
+   */
+  template<typename OptimizerType>
+  typename std::enable_if<
+      traits::HasMaxIterationsSignature<OptimizerType>::value, void>::type
+  PrintMaxIterations(const OptimizerType& optimizer, std::stringstream& stream)
+  {
+    PrettyPrintElement(stream, "Maximum iterations:", 30);
+    stream << optimizer.MaxIterations() << std::endl;
+
+    PrettyPrintElement(stream, "Reached maximum iterations:", 30);
+    stream << std::string(optimizer.MaxIterations() == objectives.size() ?
+        "true" : "false") << std::endl;
+  }
+
+  template<typename OptimizerType>
+  typename std::enable_if<
+      !traits::HasMaxIterationsSignature<OptimizerType>::value, void>::type
+  PrintMaxIterations(const OptimizerType& /* optimizer */,
+                     std::stringstream& /* stream */) { }
+
+  /**
+   * Helper function to output the batch-size to the specified output stream.
+   *
+   * @param optimizer The instantiated optimizer that implements BatchSize().
+   * @param stream The output stream.
+   */
+  template<typename OptimizerType>
+  typename std::enable_if<traits::HasBatchSizeSignature<OptimizerType>::value,
+      void>::type
+  PrintBatchSize(const OptimizerType& optimizer, std::stringstream& stream)
+  {
+    PrettyPrintElement(stream, "Batch size:", 30);
+    stream << optimizer.BatchSize() << std::endl;
+  }
+
+  template<typename OptimizerType>
+  typename std::enable_if<!traits::HasBatchSizeSignature<OptimizerType>::value,
+      void>::type
+  PrintBatchSize(const OptimizerType& /* optimizer */,
+                 std::stringstream& /* stream */) { }
+
+  /**
+   * Output formatted data.
+   *
+   * @param out Output stream.
+   * @param data The data to print on the given stream.
+   * @param width The width of the the formatted output data.
+   */
+  template<typename T>
+  void PrettyPrintElement(std::ostream& out,
+                          const T& data,
+                          const size_t width = 14)
+  {
+    out << std::left << std::setw(width) << std::setfill(' ')
+        << std::setprecision(3) << data;
+  }
+
+  /**
+   * Output formatted data.
+   *
+   * @param data The data to print on the given stream.
+   * @param width The width of the the formatted output data.
+   */
+  template<typename T>
+  void PrettyPrintElement(const T& data, const size_t width = 14)
+  {
+    PrettyPrintElement(output, data, width);
+  }
+
+  /**
+   * Outputs the given matrix in a truncated format. For example, the matrix:
+   *
+   * 1 2 3 4 5
+   * 6 7 8 9 10
+   * 11 12 13 14
+   * 15 16 17 18
+   *
+   * will be truncated to:
+   *
+   * 1 2 ... 5
+   * 6 7 ... 10
+   * ...
+   * 15 16 ... 18
+   *
+   * @param data The data to print on the given stream in a truncated format.
+   * @param size The number of elements per column/row.
+   */
+  template<typename T>
+  void TruncatePrint(const T& data, const size_t size)
+  {
+    // We can't directly output the result of submat or use .print, because
+    // both introduce a new line at the end, so we iterate over the elements.
+    for (size_t c = 0, n = 0; c < data.n_cols; ++c)
+    {
+      // Skip to the last column.
+      if (c >= (size - 1))
+      {
+        output << "..." << std::endl;
+        n = (data.n_cols - 2) * data.n_rows - 1;
+      }
+
+      for (size_t r = 0; r < data.n_rows; ++r)
+      {
+        // Check if need to skip to the last row.
+        if (r < (size - 1))
+        {
+          output << std::fixed;
+
+          // Add space for positive value, to align with negative values.
+          if (data(n) >= 0)
+            output << " ";
+
+          output << data(n++) << " ";
+        }
+        else
+        {
+          n = (c + 1) * data.n_rows - 1;
+          output << " ... " << data(n) << std::endl;
+          break;
+        }
+      }
+
+      if (c >= (size - 1))
+        break;
+    }
+  }
+
+  /**
+   * Helper function to store the step-size.
+   *
+   * @param optimizer The instantiated optimzer that implements StepSize().
+   */
+  template<typename OptimizerType>
+  typename std::enable_if<traits::HasStepSizeSignature<OptimizerType>::value,
+      void>::type
+  SaveStepSize(const OptimizerType& optimizer)
+  {
+    stepsizes.push_back(optimizer.StepSize());
+  }
+
+  template<typename OptimizerType>
+  typename std::enable_if<!traits::HasStepSizeSignature<OptimizerType>::value,
+      void>::type
+  SaveStepSize(const OptimizerType& /* optimizer */) { }
+
+  //! The number of iterations to print in percent.
+  double iterationsPercentage;
+
+  //! The output stream that all data is to be sent to; example: std::cout.
+  std::ostream& output;
+
+  //! The number of values to print for the function coordinates.
+  size_t outputMatrixSize;
+  //! The initial coordinates.
+  arma::mat initialCoordinates;
+
+  //! Gradient norm storage.
+  std::vector<double> gradientsNorm;
+
+  //! Objective storage.
+  std::vector<double> objectives;
+
+  //! Timing storage.
+  std::vector<double> timings;
+
+  //! Step-size storage.
+  std::vector<double> stepsizes;
+
+  //! Objective over the current epoch.
+  double objective;
+
+  //! Locally-stored gradient norm for a single step.
+  double gradientNorm;
+
+  //! Whether Gradient() was called.
+  bool hasGradient;
+
+  //! Whether EndEpoch() was called.
+  bool hasEndEpoch;
+
+  //! The number of Gradient() calls.
+  size_t gradientCalls;
+
+  //! The number of Evaluate() calls.
+  size_t evaluateCalls;
+
+  //! The number of BeginEpoch() calls.
+  size_t epochCalls;
+
+  //! Locally-stored optimization step timer object.
+  arma::wall_clock optimizationTimer;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/store_best_coordinates.hpp

@@ -55,7 +55,7 @@ class StoreBestCoordinates
   //! Get the best coordinates.
   ModelMatType const& BestCoordinates() const { return bestCoordinates; }
   //! Modify the best coordinates.
-  ModelMatType& BestCoordinatesl() { return bestCoordinates; }
+  ModelMatType& BestCoordinates() { return bestCoordinates; }
 
   //! Get the best objective.
   double const& BestObjective() const { return bestObjective; }

include/ensmallen_bits/callbacks/traits.hpp

@@ -37,12 +37,8 @@ ENS_HAS_EXACT_METHOD_FORM(BeginEpoch, HasBeginEpoch)
 ENS_HAS_EXACT_METHOD_FORM(EndEpoch, HasEndEpoch)
 //! Detect an StepTaken() method.
 ENS_HAS_EXACT_METHOD_FORM(StepTaken, HasStepTaken)
-//! Detect an BatchSize() method.
-ENS_HAS_EXACT_METHOD_FORM(BatchSize, HasBatchSize)
-//! Detect an MaxIterations() method.
-ENS_HAS_EXACT_METHOD_FORM(MaxIterations, HasMaxIterations)
-//! Detect an NumFunctions() method.
-ENS_HAS_EXACT_METHOD_FORM(NumFunctions, HasNumFunctions)
+//! Detect an GenerationalStepTaken() method.
+ENS_HAS_EXACT_METHOD_FORM(GenerationalStepTaken, HasGenerationalStepTaken)
 
 template<typename OptimizerType,
          typename FunctionType,
@@ -92,6 +88,15 @@ struct TypedForms
                             const MatType&,
                             const MatType&);
 
+  //! This is the form of a bool Gradient() callback method where the gradient
+  //! is modifiable.
+  template<typename CallbackType>
+  using GradientBoolModifiableForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            MatType&);
+
   //! This is the form of a void Gradient() callback method.
   template<typename CallbackType>
   using GradientVoidForm =
@@ -100,6 +105,15 @@ struct TypedForms
                             const MatType&,
                             const MatType&);
 
+  //! This is the form of a void Gradient() callback method where the gradient
+  //! is modifiable.
+  template<typename CallbackType>
+  using GradientVoidModifiableForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            MatType&);
+
   //! This is the form of a bool GradientConstraint() callback method.
   template<typename CallbackType>
   using GradientConstraintBoolForm =
@@ -109,6 +123,16 @@ struct TypedForms
                             const size_t,
                             const MatType&);
 
+  //! This is the form of a bool GradientConstraint() callback method where the
+  //! gradient is modifiable.
+  template<typename CallbackType>
+  using GradientConstraintBoolModifiableForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            MatType&);
+
   //! This is the form of a void GradientConstraint() callback method.
   template<typename CallbackType>
   using GradientConstraintVoidForm =
@@ -118,6 +142,16 @@ struct TypedForms
                             const size_t,
                             const MatType&);
 
+  //! This is the form of a void GradientConstraint() callback method where the
+  //! gradient is modifiable.
+  template<typename CallbackType>
+  using GradientConstraintVoidModifiableForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            MatType&);
+
   //! This is the form of a bool BeginOptimization() callback method.
   template<typename CallbackType>
   using BeginOptimizationBoolForm =
@@ -234,9 +268,9 @@ struct HasEvaluateSignature
 {
   const static bool value =
       HasEvaluate<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EvaluateBoolForm>::value ||
+          FunctionType, MatType>::template EvaluateBoolForm>::value ||
       HasEvaluate<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EvaluateVoidForm>::value;
+          FunctionType, MatType>::template EvaluateVoidForm>::value;
 };
 
 //! Utility struct, check if either void EvaluateConstraint() or
@@ -249,9 +283,9 @@ struct HasEvaluateConstraintSignature
 {
   const static bool value =
       HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EvaluateConstraintBoolForm>::value ||
+          FunctionType, MatType>::template EvaluateConstraintBoolForm>::value ||
       HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EvaluateConstraintVoidForm>::value;
+          FunctionType, MatType>::template EvaluateConstraintVoidForm>::value;
 };
 
 //! Utility struct, check if either void Gradient() or bool Gradient()
@@ -265,9 +299,15 @@ struct HasGradientSignature
 {
   const static bool value =
       HasGradient<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType, Gradient>::template GradientBoolForm>::value ||
+          FunctionType, MatType, Gradient>::template GradientBoolForm>::value ||
       HasGradient<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType, Gradient>::template GradientVoidForm>::value;
+          FunctionType, MatType,
+          Gradient>::template GradientBoolModifiableForm>::value ||
+      HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template GradientVoidForm>::value ||
+      HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType,
+          Gradient>::template GradientVoidModifiableForm>::value;
 };
 
 //! Utility struct, check if either void GradientConstraint() or
@@ -371,44 +411,69 @@ struct HasStepTakenSignature
          FunctionType, MatType>::template StepTakenVoidForm>::value;
 };
 
-//! Utility struct, check if size_t BatchSize() const exists.
-template<typename OptimizerType>
-struct HasBatchSizeSignature
+//! A utility struct for Typed Forms required in
+//! callbacks for MultiObjective Optimizers.
+template<typename OptimizerType,
+         typename FunctionType,
+         typename MatType,
+         typename ObjectivesVecType,
+         typename IndicesType,
+         typename GradType = MatType>
+struct MOOTypedForms
 {
-  template<typename C>
-  using BatchSizeForm = size_t(C::*)(void) const;
+  //! This is the form of a bool GenerationalStepTaken() for MOO callback method.
+  template<typename CallbackType>
+  using GenerationalStepTakenBoolForm =
+      bool(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const ObjectivesVecType&,
+                            const IndicesType&);
 
-  const static bool value =
-      HasBatchSize<OptimizerType, BatchSizeForm>::value;
+  //! This is the form of a void StepTaken() for MOO callback method.
+  template<typename CallbackType>
+  using GenerationalStepTakenVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const ObjectivesVecType&,
+                            const IndicesType&);
 };
 
-//! Utility struct, check if size_t MaxIterations() const exists.
-template<typename OptimizerType>
-struct HasMaxIterationsSignature
+//! Utility struct, check if either void StepTaken() or bool StepTaken() exists.
+//! Specialization for Multiobjective case.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename ObjectivesVecType,
+         typename IndicesType,
+         typename MatType>
+ struct HasGenerationalStepTakenSignature
 {
-  template<typename C>
-  using HasMaxIterationsForm = size_t(C::*)(void) const;
+  const static bool hasBool =
+    HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenBoolForm>::value &&
+    !HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenVoidForm>::value;
 
-  const static bool value =
-      HasMaxIterations<OptimizerType, HasMaxIterationsForm>::value;
+  const static bool hasVoid =
+    !HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenBoolForm>::value &&
+    HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenVoidForm>::value;
+
+  const static bool hasNone =
+    !HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenBoolForm>::value &&
+    !HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenVoidForm>::value;
 };
-
-//! Utility struct, check if size_t NumFunctions() const or
-//! size_t NumFunctions() exists.
-template<typename OptimizerType>
-struct HasNumFunctionsSignature
-{
-  template<typename C>
-  using NumFunctionsConstForm = size_t(C::*)(void) const;
-
-  template<typename C>
-  using NumFunctionsForm = size_t(C::*)(void);
-
-  const static bool value =
-      HasNumFunctions<OptimizerType, NumFunctionsForm>::value ||
-      HasNumFunctions<OptimizerType, NumFunctionsConstForm>::value;
-};
-
 } // namespace traits
 } // namespace callbacks
 } // namespace ens

include/ensmallen_bits/cmaes/cmaes.hpp

@@ -97,9 +97,9 @@ class CMAES
                                        MatType& iterate,
                                        CallbackTypes&&... callbacks);
 
-  //! Get the step size.
+  //! Get the population size.
   size_t PopulationSize() const { return lambda; }
-  //! Modify the step size.
+  //! Modify the population size.
   size_t& PopulationSize() { return lambda; }
 
   //! Get the lower bound of decision variables.

include/ensmallen_bits/cmaes/cmaes_impl.hpp

@@ -77,7 +77,7 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
   const double muEffective = 1 / arma::accu(arma::pow(w, 2));
 
   // Step size control parameters.
-  BaseMatType sigma(3, 1); // sigma is vector-shaped.
+  BaseMatType sigma(2, 1); // sigma is vector-shaped.
   sigma(0) = 0.3 * (upperBound - lowerBound);
   const double cs = (muEffective + 2) / (iterate.n_elem + muEffective + 5);
   const double ds = 1 + cs + 2 * std::max(std::sqrt((muEffective - 1) /
@@ -97,7 +97,7 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
       muEffective) / (std::pow(iterate.n_elem + 2, 2) +
       alphaMu * muEffective / 2));
 
-  std::vector<BaseMatType> mPosition(3, BaseMatType(iterate.n_rows,
+  std::vector<BaseMatType> mPosition(2, BaseMatType(iterate.n_rows,
       iterate.n_cols));
   mPosition[0] = lowerBound + arma::randu<BaseMatType>(
       iterate.n_rows, iterate.n_cols) * (upperBound - lowerBound);
@@ -159,7 +159,7 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
     // add a small value and try again.
     BaseMatType covLower;
     while (!arma::chol(covLower, C[idx0], "lower"))
-      C[idx0].diag() += 1e-16;
+      C[idx0].diag() += std::numeric_limits<ElemType>::epsilon();
 
     for (size_t j = 0; j < lambda; ++j)
     {
@@ -216,8 +216,7 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
     }
 
     const ElemType psNorm = arma::norm(ps[idx1]);
-    sigma(idx1) = sigma(idx0) * std::pow(
-        std::exp(cs / ds * psNorm / enn - 1), 0.3);
+    sigma(idx1) = sigma(idx0) * std::exp(cs / ds * ( psNorm / enn - 1));
 
     // Update covariance matrix.
     if ((psNorm / sqrt(1 - std::pow(1 - cs, 2 * i))) < h)

include/ensmallen_bits/cne/cne_impl.hpp

@@ -93,7 +93,7 @@ typename MatType::elem_type CNE::Optimize(ArbitraryFunctionType& function,
   std::vector<BaseMatType> population;
   for (size_t i = 0 ; i < populationSize; ++i)
   {
-    population.push_back(arma::randu<BaseMatType>(iterate.n_rows,
+    population.push_back(arma::randn<BaseMatType>(iterate.n_rows,
         iterate.n_cols) + iterate);
   }
 

include/ensmallen_bits/de/de.hpp

@@ -43,12 +43,14 @@ namespace ens {
  *
  * For more information, see the following:
  *
+ * @code
  * @techreport{storn1995,
  *   title    = {Differential Evolution—a simple and efficient adaptive scheme
  *               for global optimization over continuous spaces},
  *   author   = {Storn, Rainer and Price, Kenneth},
  *   year     = 1995
  * }
+ * @endcode
  *
  * DE can optimize arbitrary functions.  For more details, see the
  * documentation on function types included with this distribution or on the

include/ensmallen_bits/ens_version.hpp

@@ -15,13 +15,17 @@
 #define ENS_VERSION_MAJOR 2
 // The minor version is two digits so regular numerical comparisons of versions
 // work right.  The first minor version of a release is always 10.
-#define ENS_VERSION_MINOR 10
-#define ENS_VERSION_PATCH 5
+#define ENS_VERSION_MINOR 18
+#define ENS_VERSION_PATCH 2
 // If this is a release candidate, it will be reflected in the version name
 // (i.e. the version name will be "RC1", "RC2", etc.).  Otherwise the version
 // name will typically be a seemingly arbitrary set of words that does not
 // contain the capitalized string "RC".
-#define ENS_VERSION_NAME "Fried Chicken"
+#define ENS_VERSION_NAME "Fairmount Bagel"
+// Incorporate the date the version was released.
+#define ENS_VERSION_YEAR "2022"
+#define ENS_VERSION_MONTH "02"
+#define ENS_VERSION_DAY "13"
 
 namespace ens {
 
@@ -41,6 +45,14 @@ struct version
 
     return ss.str();
   }
+    
+  static inline std::string date()
+  {
+    std::stringstream ss;
+    ss << ENS_VERSION_YEAR << '-' << ENS_VERSION_MONTH << '-' << ENS_VERSION_DAY;
+
+    return ss.str();
+  }
 };
 
 } // namespace ens

include/ensmallen_bits/eve/eve.hpp

@@ -31,6 +31,7 @@ namespace ens {
  *   year    = {2016},
  *   url     = {http://arxiv.org/abs/1611.01505}
  * }
+ * @endcode
  *
  * For Eve to work, a SeparableFunctionType template parameter is required.
  * This class must implement the following function:
@@ -88,9 +89,9 @@ class Eve
       const bool exactObjective = false);
 
   /**
-   * Optimize the given function using stochastic gradient descent.  The given
-   * starting point will be modified to store the finishing point of the
-   * algorithm, and the final objective value is returned.
+   * Optimize the given function using Eve. The given starting point will be
+   * modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
    *
    * @tparam SeparableFunctionType Type of the function to be optimized.
    * @tparam MatType Type of the parameters matrix.

include/ensmallen_bits/ftml/ftml.hpp

@@ -105,7 +105,8 @@ class FTML
            CallbackTypes&&... callbacks)
   {
     return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
-        CallbackTypes...>(function, iterate, callbacks...);
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
   }
 
   //! Forward the MatType as GradType.

include/ensmallen_bits/function/arma_traits.hpp

@@ -76,6 +76,31 @@ struct MatTypeTraits<arma::SpSubview<eT>>
       "or a matrix alias instead!");
 };
 
+
+#if ((ARMA_VERSION_MAJOR >= 10) || \
+    ((ARMA_VERSION_MAJOR == 9) && (ARMA_VERSION_MINOR >= 869)))
+
+// Armadillo 9.869+ has SpSubview_col and SpSubview_row
+
+template<typename eT>
+struct MatTypeTraits<arma::SpSubview_col<eT>>
+{
+  static_assert(sizeof(arma::SpSubview_col<eT>) == 0,
+      "Armadillo subviews cannot be passed to Optimize()!  Create a matrix "
+      "or a matrix alias instead!");
+};
+
+template<typename eT>
+struct MatTypeTraits<arma::SpSubview_row<eT>>
+{
+  static_assert(sizeof(arma::SpSubview_row<eT>) == 0,
+      "Armadillo subviews cannot be passed to Optimize()!  Create a matrix "
+      "or a matrix alias instead!");
+};
+
+#endif
+
+
 template<typename eT>
 struct MatTypeTraits<arma::Cube<eT>>
 {

include/ensmallen_bits/function/static_checks.hpp

@@ -387,6 +387,25 @@ inline void CheckArbitraryFunctionTypeAPI()
 #endif
 }
 
+template<typename FunctionType, typename... RemainingTypes>
+typename std::enable_if<(sizeof...(RemainingTypes) > 1), void>::type
+CheckArbitraryFunctionTypeAPI()
+{
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  constexpr size_t size = sizeof...(RemainingTypes);
+  using TupleType = typename std::tuple<RemainingTypes...>;
+  using MatType = typename std::tuple_element<size - 1, TupleType>::type;
+
+  static_assert(CheckEvaluate<FunctionType, MatType, MatType>::value,
+      "One of the provided FunctionType does not have a correct definition of Evaluate(). "
+      "Please check that the corresponding FunctionType fully satisfies the requirements "
+      "of the ArbitraryFunctionType API; see the optimizer tutorial for "
+      "more details.");
+
+  CheckArbitraryFunctionTypeAPI<RemainingTypes...>();
+#endif
+}
+
 /**
  * Perform checks for the ResolvableFunctionType API.
  */

include/ensmallen_bits/function/traits.hpp

@@ -39,6 +39,14 @@ ENS_HAS_EXACT_METHOD_FORM(GradientConstraint, HasGradientConstraint)
 ENS_HAS_EXACT_METHOD_FORM(NumFeatures, HasNumFeatures)
 //! Detect a PartialGradient() method.
 ENS_HAS_EXACT_METHOD_FORM(PartialGradient, HasPartialGradient)
+//! Detect an MaxIterations() method.
+ENS_HAS_EXACT_METHOD_FORM(MaxIterations, HasMaxIterations)
+//! Detect an ResetPolicy() method.
+ENS_HAS_EXACT_METHOD_FORM(ResetPolicy, HasResetPolicy)
+//! Detect an BatchSize() method.
+ENS_HAS_EXACT_METHOD_FORM(BatchSize, HasBatchSize)
+//! Detect an StepSize() method.
+ENS_HAS_EXACT_METHOD_FORM(StepSize, HasStepSize)
 
 template<typename MatType, typename GradType>
 struct TypedForms
@@ -369,6 +377,76 @@ struct HasConstSignatures
   const static bool value = HasEitherConstForm && HasAnyFormA && HasAnyFormB;
 };
 
+//! Utility struct, check if size_t BatchSize() const or size_t BatchSize()
+//! exists.
+template<typename OptimizerType>
+struct HasBatchSizeSignature
+{
+  template<typename C>
+  using BatchSizeConstForm = size_t(C::*)(void) const;
+
+  template<typename C>
+  using BatchSizeForm = size_t(C::*)(void);
+
+  const static bool value =
+      HasBatchSize<OptimizerType, BatchSizeForm>::value ||
+      HasBatchSize<OptimizerType, BatchSizeConstForm>::value;
+};
+
+//! Utility struct, check if size_t StepSize() const or size_t StepSize()
+//! exists.
+template<typename OptimizerType>
+struct HasStepSizeSignature
+{
+  template<typename C>
+  using StepSizeConstForm = double(C::*)(void) const;
+
+  template<typename C>
+  using StepSizeForm = double(C::*)(void);
+
+  const static bool value =
+      HasStepSize<OptimizerType, StepSizeForm>::value ||
+      HasStepSize<OptimizerType, StepSizeConstForm>::value;
+};
+
+//! Utility struct, check if size_t MaxIterations() const exists.
+template<typename OptimizerType>
+struct HasMaxIterationsSignature
+{
+  template<typename C>
+  using HasMaxIterationsForm = size_t(C::*)(void) const;
+
+  const static bool value =
+      HasMaxIterations<OptimizerType, HasMaxIterationsForm>::value;
+};
+
+//! Utility struct, check if size_t NumFunctions() const or
+//! size_t NumFunctions() exists.
+template<typename OptimizerType>
+struct HasNumFunctionsSignature
+{
+  template<typename C>
+  using NumFunctionsConstForm = size_t(C::*)(void) const;
+
+  template<typename C>
+  using NumFunctionsForm = size_t(C::*)(void);
+
+  const static bool value =
+      HasNumFunctions<OptimizerType, NumFunctionsForm>::value ||
+      HasNumFunctions<OptimizerType, NumFunctionsConstForm>::value;
+};
+
+//! Utility struct, check if bool ResetPolicy() exists.
+template<typename OptimizerType>
+struct HasResetPolicySignature
+{
+  template<typename C>
+  using HasResetPolicyForm = bool&(C::*)(void);
+
+  const static bool value =
+      HasResetPolicy<OptimizerType, HasResetPolicyForm>::value;
+};
+
 } // namespace traits
 } // namespace ens
 

include/ensmallen_bits/fw/proximal/proximal_impl.hpp

@@ -45,7 +45,7 @@ inline void Proximal::ProjectToL1Ball(MatType& v, double tau)
   MatType simplexSum = arma::cumsum(simplexSol);
 
   double nu = 0;
-  size_t rho = 0;
+  size_t rho = simplexSol.n_rows - 1;
   for (size_t j = 1; j <= simplexSol.n_rows; j++)
   {
     rho = simplexSol.n_rows - j;

include/ensmallen_bits/lbfgs/lbfgs_impl.hpp

@@ -391,7 +391,7 @@ L_BFGS::Optimize(FunctionType& function,
     //
     // But don't do this on the first iteration to ensure we always take at
     // least one descent step.
-    if (itNum > 0 && (arma::norm(gradient, 2) < minGradientNorm))
+    if (arma::norm(gradient, 2) < minGradientNorm)
     {
       Info << "L-BFGS gradient norm too small (terminating successfully)."
           << std::endl;

include/ensmallen_bits/lookahead/lookahead.hpp

@@ -0,0 +1,259 @@
+/**
+ * @file lookahead.hpp
+ * @author Marcus Edel
+ *
+ * Lookahead is a stochastic gradient based optimization method which chooses a
+ * search direction by looking ahead at the sequence of "fast weights" generated
+ * by another optimizer.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_LOOKAHEAD_LOOKAHEAD_HPP
+#define ENSMALLEN_LOOKAHEAD_LOOKAHEAD_HPP
+
+#include <ensmallen_bits/adam/adam.hpp>
+#include <ensmallen_bits/sgd/decay_policies/no_decay.hpp>
+
+namespace ens {
+
+/**
+ * Lookahead is a stochastic gradient based optimization method which chooses a
+ * search direction by looking ahead at the sequence of "fast weights" generated
+ * by another optimizer.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @article{Zhang2019,
+ *   author  = {Michael R. Zhang and James Lucas and Geoffrey E. Hinton and
+ *              Jimmy Ba},
+ *   title   = {Lookahead Optimizer: k steps forward, 1 step back},
+ *   journal = {CoRR},
+ *   year    = {2019},
+ *   url     = {http://arxiv.org/abs/1907.08610}
+ * }
+ * @endcode
+ *
+ * Lookahead can optimize differentiable separable functions.  For more details,
+ * see the documentation on function types included with this distribution or on
+ * the ensmallen website.
+ *
+ * @tparam BaseOptimizerType Optimizer type for the forward step. By default the
+ *     Adam optimizer is used.
+ * @tparam DecayPolicyType Decay policy used during the iterative update
+ *     process to adjust the step size. By default the step size isn't going to
+ *     be adjusted (i.e. NoDecay is used).
+ */
+template<typename BaseOptimizerType = Adam, typename DecayPolicyType = NoDecay>
+class Lookahead
+{
+ public:
+  /**
+   * Construct the Lookahead optimizer with the given function, parameters
+   * and the default Adam optimizer for the forward step. The defaults here are
+   * not necessarily good for the given problem, so it is suggested that the
+   * values used be tailored to the task at hand.  The maximum number of
+   * iterations refers to the maximum number of points that are processed
+   * (i.e., one iteration equals one point; one iteration does not equal
+   * one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param k The synchronization period.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *        limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param decayPolicy Instantiated decay policy used to adjust the step size.
+   * @param resetPolicy Flag that determines whether update policy parameters
+   *                    are reset before every outer Optimize call.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
+   */
+  Lookahead(const double stepSize = 0.5,
+            const size_t k = 5,
+            const size_t maxIterations = 100000,
+            const double tolerance = 1e-5,
+            const DecayPolicyType& decayPolicy = DecayPolicyType(),
+            const bool resetPolicy = false,
+            const bool exactObjective = false);
+
+  /**
+   * Construct the Lookahead optimizer with the given function and parameters.
+   * The defaults here are not necessarily good for the given problem, so it is
+   * suggested that the values used be tailored to the task at hand.  The
+   * maximum number of iterations refers to the maximum number of points that
+   * are processed (i.e., one iteration equals one point; one iteration does not
+   * equal one pass over the dataset).
+   *
+   * @param baseOptimizer Optimizer for the forward step.
+   * @param stepSize Step size for each iteration.
+   * @param k The synchronization period.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *        limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param decayPolicy Instantiated decay policy used to adjust the step size.
+   * @param resetPolicy Flag that determines whether update policy parameters
+   *                    are reset before every outer Optimize call.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
+   */
+  Lookahead(const BaseOptimizerType& baseOptimizer,
+            const double stepSize = 0.5,
+            const size_t k = 5,
+            const size_t maxIterations = 100000,
+            const double tolerance = 1e-5,
+            const DecayPolicyType& decayPolicy = DecayPolicyType(),
+            const bool resetPolicy = false,
+            const bool exactObjective = false);
+
+  /**
+   * Clean any memory associated with the Lookahead object.
+   */
+  ~Lookahead();
+
+  /**
+   * Optimize the given function using Lookahead. The given starting point will
+   * be modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of the parameters matrix.
+   * @tparam GradType Type of the gradient matrix.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the base optimizer.
+  BaseOptimizerType BaseOptimizer() const { return baseOptimizer; }
+  //! Modify the base optimizer.
+  BaseOptimizerType& BaseOptimizer() { return baseOptimizer; }
+
+  //! Get the step size.
+  double StepSize() const { return stepSize; }
+  //! Modify the step size.
+  double& StepSize() { return stepSize; }
+
+  //! Get the synchronization period.
+  size_t K() const { return k; }
+  //! Modify the synchronization period.
+  size_t& K() { return k; }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return maxIterations; }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return maxIterations; }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return tolerance; }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return tolerance; }
+
+  //! Get the step size decay policy.
+  const DecayPolicyType& DecayPolicy() const { return decayPolicy; }
+  //! Modify the step size decay policy.
+  DecayPolicyType& DecayPolicy() { return decayPolicy; }
+
+  //! Get the instantiated decay policy type.  Be sure to check its type with
+  //! Has() before using!
+  const Any& InstDecayPolicy() const { return instDecayPolicy; }
+  //! Modify the instantiated decay policy type.  Be sure to check its type with
+  //! Has() before using!
+  Any& InstDecayPolicy() { return instDecayPolicy; }
+
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return exactObjective; }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return exactObjective; }
+
+ private:
+  /**
+   * Set the maximum number of iterations if the given optimizer implements
+   * MaxIterations().
+   *
+   * @param optimizer Optimizer to check for MaxIterations().
+   * @param k The number of iterations.
+   */
+  template<typename OptimizerType>
+  static typename std::enable_if<traits::HasMaxIterationsSignature<
+      OptimizerType>::value, void>::type
+  SetMaxIterations(OptimizerType& optimizer, const size_t k)
+  {
+    optimizer.MaxIterations() = k;
+  }
+
+  template<typename OptimizerType>
+  static typename std::enable_if<!traits::HasMaxIterationsSignature<
+      OptimizerType>::value, void>::type
+  SetMaxIterations(const OptimizerType& /* optimizer */, const size_t /* k */)
+  {
+    Warn << "The base optimizer does not have a definition of "
+        << "MaxIterations(), the base optimizer will have its configuration "
+        << "unchanged.";
+  }
+
+  //! The base optimizer for the forward step.
+  BaseOptimizerType baseOptimizer;
+
+  //! The step size for each example.
+  double stepSize;
+
+  //! Synchronization period.
+  size_t k;
+
+  //! The maximum number of allowed iterations.
+  size_t maxIterations;
+
+  //! The tolerance for termination.
+  double tolerance;
+
+  //! The decay policy used to update the step size.
+  DecayPolicyType decayPolicy;
+
+  //! Flag indicating whether update policy
+  //! should be reset before running the outer optimization.
+  bool resetPolicy;
+
+  //! Controls whether or not the actual Objective value is calculated.
+  bool exactObjective;
+
+  //! Flag indicating whether the update policy
+  //! parameters have been initialized.
+  bool isInitialized;
+
+  //! The initialized decay policy.
+  Any instDecayPolicy;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "lookahead_impl.hpp"
+
+#endif

include/ensmallen_bits/lookahead/lookahead_impl.hpp

@@ -0,0 +1,211 @@
+/**
+ * @file lookahead_impl.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of Lookahead Optimizer: k steps forward, 1 step back.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_LOOKAHEAD_LOOKAHEAD_IMPL_HPP
+#define ENSMALLEN_LOOKAHEAD_LOOKAHEAD_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "lookahead.hpp"
+
+#include <ensmallen_bits/function.hpp>
+
+namespace ens {
+
+template<typename BaseOptimizerType, typename DecayPolicyType>
+inline Lookahead<BaseOptimizerType, DecayPolicyType>::Lookahead(
+    const double stepSize,
+    const size_t k,
+    const size_t maxIterations,
+    const double tolerance,
+    const DecayPolicyType& decayPolicy,
+    const bool resetPolicy,
+    const bool exactObjective) :
+    baseOptimizer(BaseOptimizerType()),
+    stepSize(stepSize),
+    k(k),
+    maxIterations(maxIterations),
+    tolerance(tolerance),
+    decayPolicy(decayPolicy),
+    resetPolicy(resetPolicy),
+    exactObjective(exactObjective),
+    isInitialized(false)
+{ /* Nothing to do. */ }
+
+template<typename BaseOptimizerType, typename DecayPolicyType>
+inline Lookahead<BaseOptimizerType, DecayPolicyType>::Lookahead(
+    const BaseOptimizerType& baseOptimizer,
+    const double stepSize,
+    const size_t k,
+    const size_t maxIterations,
+    const double tolerance,
+    const DecayPolicyType& decayPolicy,
+    const bool resetPolicy,
+    const bool exactObjective) :
+    baseOptimizer(baseOptimizer),
+    stepSize(stepSize),
+    k(k),
+    maxIterations(maxIterations),
+    tolerance(tolerance),
+    decayPolicy(decayPolicy),
+    resetPolicy(resetPolicy),
+    exactObjective(exactObjective),
+    isInitialized(false)
+{ /* Nothing to do. */ }
+
+template<typename BaseOptimizerType, typename DecayPolicyType>
+inline Lookahead<BaseOptimizerType, DecayPolicyType>::~Lookahead()
+{
+  instDecayPolicy.Clean();
+}
+
+//! Optimize the function (minimize).
+template<typename BaseOptimizerType, typename DecayPolicyType>
+template<typename SeparableFunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value,
+typename MatType::elem_type>::type
+Lookahead<BaseOptimizerType, DecayPolicyType>::Optimize(
+    SeparableFunctionType& function,
+    MatType& iterateIn,
+    CallbackTypes&&... callbacks)
+{
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
+  typedef Function<SeparableFunctionType, BaseMatType, BaseGradType>
+      FullFunctionType;
+  FullFunctionType& f(static_cast<FullFunctionType&>(function));
+
+  // The decay policy internally use a templated class so that
+  // we can know MatType and GradType only when Optimize() is called.
+  typedef typename DecayPolicyType::template Policy<BaseMatType, BaseGradType>
+      InstDecayPolicyType;
+
+  // Make sure we have all the methods that we need.
+  traits::CheckSeparableFunctionTypeAPI<FullFunctionType, BaseMatType,
+      BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  // Check if the optimizer implements HasMaxIterations() and override the
+  // parameter with k.
+  SetMaxIterations(baseOptimizer, k);
+
+  // Check if the optimizer implements ResetPolicy() and override the reset
+  // policy.
+  if (traits::HasResetPolicySignature<BaseOptimizerType>::value &&
+      baseOptimizer.ResetPolicy())
+  {
+    Warn << "Parameters are reset before every Optimize call; set "
+        << "ResetPolicy() to false.";
+    baseOptimizer.ResetPolicy() = resetPolicy;
+  }
+
+  // To keep track of where we are and how things are going.
+  ElemType overallObjective = 0;
+  ElemType lastOverallObjective = DBL_MAX;
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
+  // Initialize the decay policy if needed.
+  if (!isInitialized || !instDecayPolicy.Has<InstDecayPolicyType>())
+  {
+    instDecayPolicy.Clean();
+    instDecayPolicy.Set<InstDecayPolicyType>(
+        new InstDecayPolicyType(decayPolicy));
+    isInitialized = true;
+  }
+
+  // Now iterate!
+  terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  const size_t actualMaxIterations = (maxIterations == 0) ?
+      std::numeric_limits<size_t>::max() : maxIterations;
+  for (size_t i = 0; i < actualMaxIterations && !terminate; i++)
+  {
+    BaseMatType iterateModel = iterate;
+
+    overallObjective = baseOptimizer.Optimize(f, iterateModel,
+        callbacks...);
+
+    // Now update the learning rate if requested by the user, note we pass the
+    // latest inner model coordinates instead of the gradient.
+    instDecayPolicy.As<InstDecayPolicyType>().Update(iterate, stepSize,
+        iterateModel);
+
+    if (std::isnan(overallObjective) || std::isinf(overallObjective))
+    {
+      Warn << "Lookahead: converged to " << overallObjective
+          << "; terminating with failure.  Try a smaller step size?"
+          << std::endl;
+
+      iterate = iterateModel;
+      Callback::EndOptimization(*this, f, iterate, callbacks...);
+      return overallObjective;
+    }
+
+    if (std::abs(lastOverallObjective - overallObjective) < tolerance)
+    {
+      Info << "Lookahead: minimized within tolerance " << tolerance << "; "
+          << "terminating optimization." << std::endl;
+
+      iterate = iterateModel;
+      Callback::EndOptimization(*this, f, iterate, callbacks...);
+      return overallObjective;
+    }
+
+    iterate += stepSize * (iterateModel - iterate);
+    terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
+
+    // Save the current objective.
+    lastOverallObjective = overallObjective;
+  }
+
+  Info << "Lookahead: maximum iterations (" << maxIterations << ") reached; "
+      << "terminating optimization." << std::endl;
+
+  // Calculate final objective if exactObjective is set to true.
+  if (exactObjective)
+  {
+    // Find the number of functions to use.
+    const size_t numFunctions = f.NumFunctions();
+
+    size_t batchSize = 1;
+    // Check if the optimizer implements the BatchSize() method and use the
+    // parameter for the objective calculation.
+    if (traits::HasBatchSizeSignature<BaseOptimizerType>::value)
+      batchSize = baseOptimizer.BatchSize();
+
+    overallObjective = 0;
+    for (size_t i = 0; i < numFunctions; i += batchSize)
+    {
+      const size_t effectiveBatchSize = std::min(batchSize, numFunctions - i);
+      const ElemType objective = f.Evaluate(iterate, i, effectiveBatchSize);
+      overallObjective += objective;
+
+      Callback::Evaluate(*this, f, iterate, objective, callbacks...);
+    }
+  }
+
+  Callback::EndOptimization(*this, f, iterate, callbacks...);
+  return overallObjective;
+}
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/moead/decomposition_policies/pbi_decomposition.hpp

@@ -0,0 +1,81 @@
+/**
+ * @file pbi_decomposition.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * The Penalty Based Boundary Intersection (PBI) decomposition policy.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOEAD_PBI_HPP
+#define ENSMALLEN_MOEAD_PBI_HPP
+
+namespace ens {
+
+/**
+ * Penalty Based Boundary Intersection (PBI) method is a weight decomposition method,
+ * it tries to find the intersection between bottom-most boundary of the attainable
+ * objective set with the reference directions.
+ *
+ * The goal is to minimize the distance between objective vectors with the ideal point
+ * along the reference direction. To handle equality constraints, a penalty parameter
+ * theta is used.
+ *
+ * For more information, see the following:
+ * @code
+ * article{zhang2007moea,
+ *   title={MOEA/D: A multiobjective evolutionary algorithm based on decomposition},
+ *   author={Zhang, Qingfu and Li, Hui},
+ *   journal={IEEE Transactions on evolutionary computation},
+ *   pages={712--731},
+ *   year={2007}
+ * @endcode
+ */
+class PenaltyBoundaryIntersection
+{
+ public:
+  /**
+   * Constructor for Penalty Based Boundary Intersection decomposition
+   * policy.
+   *
+   * @param theta The penalty value.
+   */
+  PenaltyBoundaryIntersection(const double theta = 5) :
+      theta(theta)
+  {
+    /* Nothing to do. */
+  }
+
+  /**
+   * Decompose the weight vectors.
+   *
+   * @tparam VecType The type of the vector used in the decommposition.
+   * @param weight The weight vector corresponding to a subproblem.
+   * @param idealPoint The reference point in the objective space.
+   * @param candidateFitness The objective vector of the candidate.
+   */
+  template<typename VecType>
+  typename VecType::elem_type Apply(const VecType& weight,
+                                    const VecType& idealPoint,
+                                    const VecType& candidateFitness)
+  {
+    typedef typename VecType::elem_type ElemType;
+    //! A unit vector in the same direction as the provided weight vector.
+    const VecType referenceDirection = weight / arma::norm(weight);
+    //! Distance of F(x) from the idealPoint along the reference direction.
+    const ElemType d1 = arma::dot(candidateFitness - idealPoint, referenceDirection);
+    //! The perpendicular distance of F(x) from reference direction.
+    const ElemType d2 = arma::norm(candidateFitness - (idealPoint + d1 * referenceDirection));
+
+    return d1 + static_cast<ElemType>(theta) * d2;
+  }
+
+  private:
+    double theta;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/moead/decomposition_policies/tchebycheff_decomposition.hpp

@@ -0,0 +1,66 @@
+/**
+ * @file tchebycheff_decomposition.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * The Tchebycheff Weight decomposition policy.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOEAD_TCHEBYCHEFF_HPP
+#define ENSMALLEN_MOEAD_TCHEBYCHEFF_HPP
+
+namespace ens {
+
+/**
+ * The Tchebycheff method works by taking the maximum of element-wise product
+ * between reference direction and the line connecting objective vector and
+ * ideal point.
+ *
+ * Under mild conditions, for each Pareto Optimal point there exists a reference
+ * direction such that the given point is also the optimal solution
+ * to this scalar objective.
+ *
+ * For more information, see the following:
+ * @code
+ * article{zhang2007moea,
+ *   title={MOEA/D: A multiobjective evolutionary algorithm based on decomposition},
+ *   author={Zhang, Qingfu and Li, Hui},
+ *   journal={IEEE Transactions on evolutionary computation},
+ *   pages={712--731},
+ *   year={2007}
+ * @endcode
+ */
+class Tchebycheff
+{
+ public:
+  /**
+   * Constructor for Tchebycheff decomposition policy.
+   */
+  Tchebycheff()
+  {
+    /* Nothing to do. */
+  }
+
+  /**
+   * Decompose the weight vectors.
+   *
+   * @tparam VecType The type of the vector used in the decommposition.
+   * @param weight The weight vector corresponding to a subproblem.
+   * @param idealPoint The reference point in the objective space.
+   * @param candidateFitness The objective vector of the candidate.
+   */
+  template<typename VecType>
+  typename VecType::elem_type Apply(const VecType& weight,
+                                    const VecType& idealPoint,
+                                    const VecType& candidateFitness)
+  {
+    return arma::max(weight % arma::abs(candidateFitness - idealPoint));
+  }
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/moead/decomposition_policies/weighted_decomposition.hpp

@@ -0,0 +1,62 @@
+/**
+ * @file weighted_decomposition.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * The Weighted Average decomposition policy.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOEAD_WEIGHTED_HPP
+#define ENSMALLEN_MOEAD_WEIGHTED_HPP
+
+namespace ens {
+
+/**
+ * The Weighted average method of decomposition. The working principle is to
+ * minimize the dot product between reference direction and the line connecting
+ * objective vector and ideal point.
+ *
+ * For more information, see the following:
+ * @code
+ * article{zhang2007moea,
+ *   title={MOEA/D: A multiobjective evolutionary algorithm based on decomposition},
+ *   author={Zhang, Qingfu and Li, Hui},
+ *   journal={IEEE Transactions on evolutionary computation},
+ *   pages={712--731},
+ *   year={2007}
+ * @endcode
+ */
+class WeightedAverage
+{
+ public:
+  /**
+   * Constructor for Weighted Average decomposition policy.
+   */
+  WeightedAverage()
+  {
+    /* Nothing to do. */
+  }
+
+  /**
+   * Decompose the weight vectors.
+   *
+   * @tparam VecType The type of the vector used in the decommposition.
+   * @param weight The weight vector corresponding to a subproblem.
+   * @param idealPoint The reference point in the objective space.
+   * @param candidateFitness The objective vector of the candidate.
+   */
+  template<typename VecType>
+  typename VecType::elem_type Apply(const VecType& weight,
+                                    const VecType& /* idealPoint */,
+                                    const VecType& candidateFitness)
+  {
+    return arma::dot(weight, candidateFitness);
+  }
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/moead/moead.hpp

@@ -0,0 +1,338 @@
+/**
+ * @file moead.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * MOEA/D-DE is a multi objective optimization algorithm. MOEA/D-DE
+ * uses genetic algorithms along with a set of reference directions
+ * to drive the population towards the Optimal Front.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_MOEAD_MOEAD_HPP
+#define ENSMALLEN_MOEAD_MOEAD_HPP
+
+//! Decomposition policies.
+#include "decomposition_policies/tchebycheff_decomposition.hpp"
+#include "decomposition_policies/weighted_decomposition.hpp"
+#include "decomposition_policies/pbi_decomposition.hpp"
+
+//! Weight initialization policies.
+#include "weight_init_policies/uniform_init.hpp"
+#include "weight_init_policies/bbs_init.hpp"
+#include "weight_init_policies/dirichlet_init.hpp"
+
+namespace ens {
+
+/**
+ * MOEA/D-DE (Multi Objective Evolutionary Algorithm based on Decompositon - 
+ * Differential Variant) is a multiobjective optimization algorithm. This class 
+ * implements the said optimizer. 
+ *
+ * The algorithm works by generating a candidate population from a fixed starting point. 
+ * Reference directions are generated to guide the optimization process towards the Pareto Front. 
+ * Further, a decomposition function is defined to decompose the problem to a scalar optimization 
+ * objective. Utilizing genetic operators, offsprings are generated with better decomposition values 
+ * to replace the neighboring parent solutions. 
+ *
+ * For more information, see the following:
+ * @code
+ * @article{li2008multiobjective,
+ *   title={Multiobjective optimization problems with complicated Pareto sets, MOEA/D and NSGA-II},
+ *   author={Li, Hui and Zhang, Qingfu},
+ *   journal={IEEE transactions on evolutionary computation},
+ *   pages={284--302},
+ *   year={2008},
+ * @endcode
+ */
+template<typename InitPolicyType = Uniform,
+         typename DecompPolicyType = Tchebycheff>
+class MOEAD {
+ public:
+  /**
+   * Constructor for the MOEA/D optimizer.
+   *
+   * The default values provided here are not necessarily suitable for a
+   * given function. Therefore, it is highly recommended to adjust the
+   * parameters according to the problem.
+   *
+   * @param populationSize The number of elements in the population.
+   * @param maxGenerations The maximum number of generations allowed.
+   * @param crossoverProb The probability that a crossover will occur.
+   * @param neighborProb The probability of sampling from neighbor.
+   * @param neighborSize The number of nearest neighbours of weights
+   *    to find.
+   * @param distributionIndex The crowding degree of the mutation.
+   * @param differentialWeight A parameter used in the mutation of candidate
+   *     solutions controls amplification factor of the differentiation.
+   * @param maxReplace The limit of solutions allowed to be replaced by a child.
+   * @param epsilon Handle numerical stability after weight initialization.
+   * @param lowerBound The lower bound on each variable of a member
+   *    of the variable space.
+   * @param upperBound The upper bound on each variable of a member
+   *    of the variable space.
+   */
+  MOEAD(const size_t populationSize = 300,
+        const size_t maxGenerations = 500,
+        const double crossoverProb = 1.0,
+        const double neighborProb = 0.9,
+        const size_t neighborSize = 20,
+        const double distributionIndex = 20,
+        const double differentialWeight = 0.5,
+        const size_t maxReplace = 2,
+        const double epsilon = 1E-10,
+        const arma::vec& lowerBound = arma::zeros(1, 1),
+        const arma::vec& upperBound = arma::ones(1, 1),
+        const InitPolicyType initPolicy = InitPolicyType(),
+        const DecompPolicyType decompPolicy = DecompPolicyType());
+
+  /**
+   * Constructor for the MOEA/D optimizer. This constructor is provides an
+   * overload to use lowerBound and upperBound as doubles, in case all the
+   * variables in the problem have the same limits.
+   *
+   * The default values provided here are not necessarily suitable for a
+   * given function. Therefore, it is highly recommended to adjust the
+   * parameters according to the problem.
+   *
+   * @param populationSize The number of elements in the population.
+   * @param maxGenerations The maximum number of generations allowed.
+   * @param crossoverProb The probability that a crossover will occur.
+   * @param neighborProb The probability of sampling from neighbor.
+   * @param neighborSize The number of nearest neighbours of weights
+   *    to find.
+   * @param distributionIndex The crowding degree of the mutation.
+   * @param differentialWeight A parameter used in the mutation of candidate
+   *     solutions controls amplification factor of the differentiation.
+   * @param maxReplace The limit of solutions allowed to be replaced by a child.
+   * @param epsilon Handle numerical stability after weight initialization.
+   * @param lowerBound The lower bound on each variable of a member
+   *    of the variable space.
+   * @param upperBound The upper bound on each variable of a member
+   *    of the variable space.
+   */
+    MOEAD(const size_t populationSize = 300,
+          const size_t maxGenerations = 500,
+          const double crossoverProb = 1.0,
+          const double neighborProb = 0.9,
+          const size_t neighborSize = 20,
+          const double distributionIndex = 20,
+          const double differentialWeight = 0.5,
+          const size_t maxReplace = 2,
+          const double epsilon = 1E-10,
+          const double lowerBound = 0,
+          const double upperBound = 1,
+          const InitPolicyType initPolicy = InitPolicyType(),
+          const DecompPolicyType decompPolicy = DecompPolicyType());
+
+  /**
+   * Optimize a set of objectives. The initial population is generated
+   * using the initial point. The output is the best generated front.
+   *
+   * @tparam MatType The type of matrix used to store coordinates.
+   * @tparam ArbitraryFunctionType The type of objective function.
+   * @tparam CallbackTypes Types of callback function.
+   * @param objectives std::tuple of the objective functions.
+   * @param iterate The initial reference point for generating population.
+   * @param callbacks The callback functions.
+   */
+  template<typename MatType,
+           typename... ArbitraryFunctionType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(std::tuple<ArbitraryFunctionType...>& objectives,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks);
+
+  //! Retrieve population size.
+  size_t PopulationSize() const { return populationSize; }
+  //! Modify the population size.
+  size_t& PopulationSize() { return populationSize; }
+
+  //! Retrieve number of generations.
+  size_t MaxGenerations() const { return maxGenerations; }
+  //! Modify the number of generations.
+  size_t& MaxGenerations() { return maxGenerations; }
+
+  //! Retrieve crossover rate.
+  double CrossoverRate() const { return crossoverProb; }
+  //! Modify the crossover rate.
+  double& CrossoverRate() { return crossoverProb; }
+
+  //! Retrieve size of the weight neighbor.
+  size_t NeighborSize() const { return neighborSize; }
+  //! Modify the size of the weight neighbor.
+  size_t& NeighborSize() { return neighborSize; }
+
+  //! Retrieve value of the distribution index.
+  double DistributionIndex() const { return distributionIndex; }
+  //! Modify the value of the distribution index.
+  double& DistributionIndex() { return distributionIndex; }
+
+  //! Retrieve value of neighbor probability.
+  double NeighborProb() const { return neighborProb; }
+  //! Modify the value of neigbourhood probability.
+  double& NeighborProb() { return neighborProb; }
+
+  //! Retrieve value of scaling factor.
+  double DifferentialWeight() const { return differentialWeight; }
+  //! Modify the value of scaling factor.
+  double& DifferentialWeight() { return differentialWeight; }
+
+  //! Retrieve value of maxReplace.
+  size_t MaxReplace() const { return maxReplace; }
+  //! Modify value of maxReplace.
+  size_t& MaxReplace() { return maxReplace; }
+
+   //! Retrieve value of epsilon.
+  double Epsilon() const { return epsilon; }
+  //! Modify value of maxReplace.
+  double& Epsilon() { return epsilon; }
+
+  //! Retrieve value of lowerBound.
+  const arma::vec& LowerBound() const { return lowerBound; }
+  //! Modify value of lowerBound.
+  arma::vec& LowerBound() { return lowerBound; }
+
+  //! Retrieve value of upperBound.
+  const arma::vec& UpperBound() const { return upperBound; }
+  //! Modify value of upperBound.
+  arma::vec& UpperBound() { return upperBound; }
+
+  //! Retrieve the Pareto optimal points in variable space. This returns an empty cube
+  //! until `Optimize()` has been called.
+  const arma::cube& ParetoSet() const { return paretoSet; }
+
+  //! Retrieve the best front (the Pareto frontier). This returns an empty cube until
+  //! `Optimize()` has been called.
+  const arma::cube& ParetoFront() const { return paretoFront; }
+
+  //! Get the weight initialization policy.
+  const InitPolicyType& InitPolicy() const { return initPolicy; }
+  //! Modify the weight initialization policy.
+  InitPolicyType& InitPolicy() { return initPolicy; }
+
+  //! Get the weight decomposition policy.
+  const DecompPolicyType& DecompPolicy() const { return decompPolicy; }
+  //! Modify the weight decomposition policy.
+  DecompPolicyType& DecompPolicy() { return decompPolicy; }
+
+ private:
+  /**
+   * @brief Randomly selects two members from the population.
+   *
+   * @param subProblemIdx Index of the current subproblem.
+   * @param neighborSize A matrix containing indices of the neighbors.
+   * @return std::tuple<size_t, size_t> The chosen pair of indices.
+   */
+  std::tuple<size_t, size_t> Mating(size_t subProblemIdx,
+                                    const arma::umat& neighborSize,
+                                    bool sampleNeighbor);
+
+  /**
+   * Mutate the child formed by the crossover of two random members of the
+   * population. Uses polynomial mutation.
+   *
+   * @tparam MatType The type of matrix used to store coordinates.
+   * @param child The candidate to be mutated.
+   * @param mutationRate The probability of mutation.
+   * @param lowerBound The lower bound on each variable in the matrix.
+   * @param upperBound The upper bound on each variable in the matrix.
+   * @return The mutated child.
+   */
+  template<typename MatType>
+  void Mutate(MatType& child,
+              double mutationRate,
+              const MatType& lowerBound,
+              const MatType& upperBound);
+
+  /**
+   * Evaluate objectives for the elite population.
+   *
+   * @tparam ArbitraryFunctionType std::tuple of multiple function types.
+   * @tparam MatType Type of matrix to optimize.
+   * @param population The elite population.
+   * @param objectives The set of objectives.
+   * @param calculatedObjectives Vector to store calculated objectives.
+   */
+  template<std::size_t I = 0,
+           typename MatType,
+           typename ...ArbitraryFunctionType>
+  typename std::enable_if<I == sizeof...(ArbitraryFunctionType), void>::type
+  EvaluateObjectives(
+                     std::vector<MatType>&,
+                     std::tuple<ArbitraryFunctionType...>&,
+                     std::vector<arma::Col<typename MatType::elem_type> >&);
+
+  template<std::size_t I = 0,
+           typename MatType,
+           typename ...ArbitraryFunctionType>
+  typename std::enable_if<I < sizeof...(ArbitraryFunctionType), void>::type
+  EvaluateObjectives(
+                     std::vector<MatType>& population,
+                     std::tuple<ArbitraryFunctionType...>& objectives,
+                     std::vector<arma::Col<typename MatType::elem_type> >&
+                     calculatedObjectives);
+
+  //! Size of the population.
+  size_t populationSize;
+
+  //! Maximum number of generations before termination criteria is met.
+  size_t maxGenerations;
+
+  //! Probability of crossover between two members.
+  double crossoverProb;
+
+  //! The probability that two elements will be chosen from the neighbor.
+  double neighborProb;
+
+  //! Number of nearest neighbours of weights to consider.
+  size_t neighborSize;
+
+  //! The crowding degree of the mutation. Higher value produces a mutant
+  //! resembling its parent.
+  double distributionIndex;
+
+  //! Amplification factor for differentiation.
+  double differentialWeight;
+
+  //! Maximum number of childs which can replace the parent. Higher value
+  //! leads to a loss of diversity.
+  size_t maxReplace;
+
+  //! A small numeric value to be added to the weights after initialization.
+  //! Prevents zero value inside inited weights.
+  double epsilon;
+
+  //! Lower bound on each variable in the variable space.
+  arma::vec lowerBound;
+
+  //! Upper bound on each variable in the variable space.
+  arma::vec upperBound;
+
+  //! The set of all the Pareto optimal points.
+  //! Stored after Optimize() is called.
+  arma::cube paretoSet;
+
+  //! The set of all the Pareto optimal objective vectors.
+  //! Stored after Optimize() is called.
+  arma::cube paretoFront;
+
+  //! Policy to initialize the reference directions (weights) matrix.
+  InitPolicyType initPolicy;
+
+  //! Policy to decompose the weights.
+  DecompPolicyType decompPolicy;
+};
+
+using DefaultMOEAD = MOEAD<Uniform, Tchebycheff>;
+using BBSMOEAD = MOEAD<BayesianBootstrap, Tchebycheff>;
+using DirichletMOEAD = MOEAD<Dirichlet, Tchebycheff>;
+} // namespace ens
+
+// Include implementation.
+#include "moead_impl.hpp"
+
+#endif

include/ensmallen_bits/moead/moead_impl.hpp

@@ -0,0 +1,441 @@
+/**
+ * @file moead_impl.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * Implementation of the MOEA/D-DE algorithm. Used for multi-objective
+ * optimization problems on arbitrary functions.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more Information.
+ */
+
+#ifndef ENSMALLEN_MOEAD_MOEAD_IMPL_HPP
+#define ENSMALLEN_MOEAD_MOEAD_IMPL_HPP
+
+#include "moead.hpp"
+#include <assert.h>
+
+namespace ens {
+template <typename InitPolicyType, typename DecompPolicyType>
+inline MOEAD<InitPolicyType, DecompPolicyType>::
+MOEAD(const size_t populationSize,
+      const size_t maxGenerations,
+      const double crossoverProb,
+      const double neighborProb,
+      const size_t neighborSize,
+      const double distributionIndex,
+      const double differentialWeight,
+      const size_t maxReplace,
+      const double epsilon,
+      const arma::vec& lowerBound,
+      const arma::vec& upperBound,
+      const InitPolicyType initPolicy,
+      const DecompPolicyType decompPolicy) :
+    populationSize(populationSize),
+    maxGenerations(maxGenerations),
+    crossoverProb(crossoverProb),
+    neighborProb(neighborProb),
+    neighborSize(neighborSize),
+    distributionIndex(distributionIndex),
+    differentialWeight(differentialWeight),
+    maxReplace(maxReplace),
+    epsilon(epsilon),
+    lowerBound(lowerBound),
+    upperBound(upperBound),
+    initPolicy(initPolicy),
+    decompPolicy(decompPolicy)
+  { /* Nothing to do here. */ }
+
+template <typename InitPolicyType, typename DecompPolicyType>
+inline MOEAD<InitPolicyType, DecompPolicyType>::
+MOEAD(const size_t populationSize,
+      const size_t maxGenerations,
+      const double crossoverProb,
+      const double neighborProb,
+      const size_t neighborSize,
+      const double distributionIndex,
+      const double differentialWeight,
+      const size_t maxReplace,
+      const double epsilon,
+      const double lowerBound,
+      const double upperBound,
+      const InitPolicyType initPolicy,
+      const DecompPolicyType decompPolicy) :
+    populationSize(populationSize),
+    maxGenerations(maxGenerations),
+    crossoverProb(crossoverProb),
+    neighborProb(neighborProb),
+    neighborSize(neighborSize),
+    distributionIndex(distributionIndex),
+    differentialWeight(differentialWeight),
+    maxReplace(maxReplace),
+    epsilon(epsilon),
+    lowerBound(lowerBound * arma::ones(1, 1)),
+    upperBound(upperBound * arma::ones(1, 1)),
+    initPolicy(initPolicy),
+    decompPolicy(decompPolicy)
+  { /* Nothing to do here. */ }
+
+//! Optimize the function.
+template <typename InitPolicyType, typename DecompPolicyType>
+template<typename MatType,
+         typename... ArbitraryFunctionType,
+         typename... CallbackTypes>
+typename MatType::elem_type MOEAD<InitPolicyType, DecompPolicyType>::
+Optimize(std::tuple<ArbitraryFunctionType...>& objectives,
+         MatType& iterateIn,
+         CallbackTypes&&... callbacks)
+{
+  // Population Size must be at least 3 for MOEA/D-DE to work.
+  if (populationSize < 3)
+  {
+    throw std::logic_error("MOEA/D-DE::Optimize(): population size should be at least"
+        " 3!");
+  }
+
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  // Make sure that we have the methods that we need.  Long name...
+  traits::CheckArbitraryFunctionTypeAPI<ArbitraryFunctionType...,
+      BaseMatType>();
+  RequireDenseFloatingPointType<BaseMatType>();
+
+  if (neighborSize < 2)
+  {
+    throw std::invalid_argument(
+        "neighborSize should be atleast 2, however "
+        + std::to_string(neighborSize) + " was detected."
+    );
+  }
+
+  if (neighborSize > populationSize - 1u)
+  {
+    std::ostringstream oss;
+    oss << "MOEAD::Optimize(): " << "neighborSize is " << neighborSize
+        << " but populationSize is " << populationSize << "(should be"
+        << " atleast " << (neighborSize + 1u) << ")" << std::endl;
+    throw std::logic_error(oss.str());
+  }
+
+  // Check if lower bound is a vector of a single dimension.
+  if (lowerBound.n_rows == 1)
+    lowerBound = lowerBound(0, 0) * arma::ones(iterate.n_rows, iterate.n_cols);
+
+  // Check if upper bound is a vector of a single dimension.
+  if (upperBound.n_rows == 1)
+    upperBound = upperBound(0, 0) * arma::ones(iterate.n_rows, iterate.n_cols);
+
+  // Check the dimensions of lowerBound and upperBound.
+  assert(lowerBound.n_rows == iterate.n_rows && "The dimensions of "
+      "lowerBound are not the same as the dimensions of iterate.");
+  assert(upperBound.n_rows == iterate.n_rows && "The dimensions of "
+      "upperBound are not the same as the dimensions of iterate.");
+
+  const size_t numObjectives = sizeof...(ArbitraryFunctionType);
+  const size_t numVariables = iterate.n_rows;
+
+  //! Useful temporaries for float-like comparisons.
+  const BaseMatType castedLowerBound = arma::conv_to<BaseMatType>::from(lowerBound);
+  const BaseMatType castedUpperBound = arma::conv_to<BaseMatType>::from(upperBound);
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
+  // The weight matrix. Each vector represents a decomposition subproblem (M X N).
+  const BaseMatType weights = initPolicy.template Generate<BaseMatType>(
+      numObjectives, populationSize, epsilon);
+
+  // 1.1 Storing the indices of nearest neighbors of each weight vector.
+  arma::umat neighborIndices(neighborSize, populationSize);
+  for (size_t i = 0; i < populationSize; ++i)
+  {
+    // Cache the distance between weights[i] and other weights.
+    const arma::Row<ElemType> distances =
+        arma::sqrt(arma::sum(arma::square(weights.col(i) - weights.each_col())));
+    arma::uvec sortedIndices = arma::stable_sort_index(distances);
+    // Ignore distance from self.
+    neighborIndices.col(i) = sortedIndices(arma::span(1, neighborSize));
+  }
+
+  // 1.2 Random generation of the initial population.
+  std::vector<BaseMatType> population(populationSize);
+  for (BaseMatType& individual : population)
+  {
+    individual = arma::randu<BaseMatType>(
+        iterate.n_rows, iterate.n_cols) - 0.5 + iterate;
+
+    // Constrain all genes to be within bounds.
+    individual = arma::min(arma::max(individual, castedLowerBound), castedUpperBound);
+  }
+
+  Info << "MOEA/D-DE initialized successfully. Optimization started." << std::endl;
+
+  std::vector<arma::Col<ElemType>> populationFitness(populationSize);
+  std::fill(populationFitness.begin(), populationFitness.end(),
+      arma::Col<ElemType>(numObjectives, arma::fill::zeros));
+  EvaluateObjectives(population, objectives, populationFitness);
+
+  // 1.3 Initialize the ideal point z.
+  arma::Col<ElemType> idealPoint(numObjectives);
+  idealPoint.fill(std::numeric_limits<ElemType>::max());
+
+  for (const arma::Col<ElemType>& individualFitness : populationFitness)
+    idealPoint = arma::min(idealPoint, individualFitness);
+
+  terminate |= Callback::BeginOptimization(*this, objectives, iterate, callbacks...);
+
+  // 2 The main loop.
+  for (size_t generation = 1; generation <= maxGenerations && !terminate; ++generation)
+  {
+    // Shuffle indexes of subproblems.
+    const arma::uvec shuffle = arma::shuffle(
+        arma::linspace<arma::uvec>(0, populationSize - 1, populationSize));
+    for (size_t subProblemIdx : shuffle)
+    {
+      // 2.1 Randomly select two indices in neighborIndices[subProblemIdx] and use them
+      // to make a child.
+      size_t r1, r2, r3;
+      r1 = subProblemIdx;
+      // Randomly choose to sample from the population or the neighbors.
+      const bool sampleNeighbor = arma::randu() < neighborProb;
+      std::tie(r2, r3) =
+          Mating(subProblemIdx, neighborIndices, sampleNeighbor);
+
+      // 2.2 - 2.3 Reproduction and Repair: Differential Operator followed by
+      // Polynomial Mutation.
+      BaseMatType candidate(iterate.n_rows, iterate.n_cols);
+
+      for (size_t geneIdx = 0; geneIdx < numVariables; ++geneIdx)
+      {
+        if (arma::randu() < crossoverProb)
+        {
+          candidate(geneIdx) = population[r1](geneIdx) +
+              differentialWeight * (population[r2](geneIdx) -
+                  population[r3](geneIdx));
+
+          // Boundary conditions.
+          if (candidate(geneIdx) < castedLowerBound(geneIdx))
+          {
+            candidate(geneIdx) = castedLowerBound(geneIdx) +
+                arma::randu() * (population[r1](geneIdx) - castedLowerBound(geneIdx));
+          }
+          if (candidate(geneIdx) > castedUpperBound(geneIdx))
+          {
+            candidate(geneIdx) = castedUpperBound(geneIdx) -
+                arma::randu() * (castedUpperBound(geneIdx) - population[r1](geneIdx));
+          }
+        }
+        else
+          candidate(geneIdx) = population[r1](geneIdx);
+      }
+
+      Mutate(candidate, 1.0 / static_cast<double>(numVariables),
+          castedLowerBound, castedUpperBound);
+
+      arma::Col<ElemType> candidateFitness(numObjectives);
+      //! Creating temp vectors to pass to EvaluateObjectives.
+      std::vector<BaseMatType> candidateContainer { candidate };
+      std::vector<arma::Col<ElemType>> fitnessContainer { candidateFitness };
+      EvaluateObjectives(candidateContainer, objectives, fitnessContainer);
+      candidateFitness = std::move(fitnessContainer[0]);
+      //! Flush out the dummy containers.
+      fitnessContainer.clear();
+      candidateContainer.clear();
+
+      // 2.4 Update of ideal point.
+      idealPoint = arma::min(idealPoint, candidateFitness);
+
+      // 2.5 Update of the population.
+      size_t replaceCounter = 0;
+      const size_t sampleSize = sampleNeighbor ? neighborSize : populationSize;
+
+      const arma::uvec idxShuffle = arma::shuffle(
+          arma::linspace<arma::uvec>(0, sampleSize - 1, sampleSize));
+
+      for (size_t idx : idxShuffle)
+      {
+        // Preserve diversity by controlling replacement of neighbors
+        // by child solution.
+        if (replaceCounter >= maxReplace)
+          break;
+
+        const size_t pick = sampleNeighbor ?
+            neighborIndices(idx, subProblemIdx) : idx;
+
+        const ElemType candidateDecomposition = decompPolicy.template
+            Apply<arma::Col<ElemType>>(weights.col(pick), idealPoint, candidateFitness);
+        const ElemType parentDecomposition =  decompPolicy.template
+            Apply<arma::Col<ElemType>>(weights.col(pick), idealPoint, populationFitness[pick]);
+
+        if (candidateDecomposition < parentDecomposition)
+        {
+          population[pick] = candidate;
+          populationFitness[pick] = candidateFitness;
+          ++replaceCounter;
+        }
+      }
+    } // End of pass over all subproblems.
+
+    //  The final population itself is the best front.
+    const std::vector<arma::uvec> frontIndices { arma::shuffle(
+        arma::linspace<arma::uvec>(0, populationSize - 1, populationSize)) };
+
+    terminate |= Callback::GenerationalStepTaken(*this, objectives, iterate,
+        populationFitness, frontIndices, callbacks...);
+  } // End of pass over all the generations.
+
+  // Set the candidates from the Pareto Set as the output.
+  paretoSet.set_size(population[0].n_rows, population[0].n_cols, population.size());
+
+  // The Pareto Front is stored, can be obtained via ParetoSet() getter.
+  for (size_t solutionIdx = 0; solutionIdx < population.size(); ++solutionIdx)
+  {
+    paretoSet.slice(solutionIdx) =
+        arma::conv_to<arma::mat>::from(population[solutionIdx]);
+  }
+
+  // Set the candidates from the Pareto Front as the output.
+  paretoFront.set_size(populationFitness[0].n_rows, populationFitness[0].n_cols,
+      populationFitness.size());
+
+  // The Pareto Front is stored, can be obtained via ParetoFront() getter.
+  for (size_t solutionIdx = 0; solutionIdx < populationFitness.size(); ++solutionIdx)
+  {
+    paretoFront.slice(solutionIdx) =
+        arma::conv_to<arma::mat>::from(populationFitness[solutionIdx]);
+  }
+
+  // Assign iterate to first element of the Pareto Set.
+  iterate = population[0];
+
+  Callback::EndOptimization(*this, objectives, iterate, callbacks...);
+
+  ElemType performance = std::numeric_limits<ElemType>::max();
+
+  for (size_t geneIdx = 0; geneIdx < numObjectives; ++geneIdx)
+  {
+    if (arma::accu(populationFitness[geneIdx]) < performance)
+      performance = arma::accu(populationFitness[geneIdx]);
+  }
+
+  return performance;
+}
+
+//! Randomly chooses to select from parents or neighbors.
+template <typename InitPolicyType, typename DecompPolicyType>
+inline std::tuple<size_t, size_t>
+MOEAD<InitPolicyType, DecompPolicyType>::
+Mating(size_t subProblemIdx,
+       const arma::umat& neighborIndices,
+       bool sampleNeighbor)
+{
+  //! Indexes of two points from the sample space.
+  size_t pointA = sampleNeighbor
+      ? neighborIndices(
+            arma::randi(arma::distr_param(0, neighborSize - 1u)), subProblemIdx)
+      : arma::randi(arma::distr_param(0, populationSize - 1u));
+
+  size_t pointB = sampleNeighbor
+      ? neighborIndices(
+            arma::randi(arma::distr_param(0, neighborSize - 1u)), subProblemIdx)
+      : arma::randi(arma::distr_param(0, populationSize - 1u));
+
+  //! If the sampled points are equal, then modify one of them
+  //! within reasonable bounds.
+  if (pointA == pointB)
+  {
+    if (pointA == populationSize - 1u)
+      --pointA;
+    else
+      ++pointA;
+  }
+
+  return std::make_tuple(pointA, pointB);
+}
+
+//! Perform Polynomial mutation of the candidate.
+template <typename InitPolicyType, typename DecompPolicyType>
+template<typename MatType>
+inline void MOEAD<InitPolicyType, DecompPolicyType>::
+Mutate(MatType& candidate,
+       double mutationRate,
+       const MatType& lowerBound,
+       const MatType& upperBound)
+{
+    const size_t numVariables = candidate.n_rows;
+    for (size_t geneIdx = 0; geneIdx < numVariables; ++geneIdx)
+    {
+      // Should this gene be mutated?
+      if (arma::randu() > mutationRate)
+        continue;
+
+      const double geneRange = upperBound(geneIdx) - lowerBound(geneIdx);
+      // Normalised distance from the bounds.
+      const double lowerDelta = (candidate(geneIdx) - lowerBound(geneIdx)) / geneRange;
+      const double upperDelta = (upperBound(geneIdx) - candidate(geneIdx)) / geneRange;
+      const double mutationPower = 1. / (distributionIndex + 1.0);
+      const double rand = arma::randu();
+      double value, perturbationFactor;
+      if (rand < 0.5)
+      {
+        value = 2.0 * rand + (1.0 - 2.0 * rand) *
+            std::pow(upperDelta, distributionIndex + 1.0);
+        perturbationFactor = std::pow(value, mutationPower) - 1.0;
+      }
+      else
+      {
+        value = 2.0 * (1.0 - rand) + 2.0 *(rand - 0.5) *
+            std::pow(lowerDelta, distributionIndex + 1.0);
+        perturbationFactor = 1.0 - std::pow(value, mutationPower);
+      }
+
+      candidate(geneIdx) += perturbationFactor * geneRange;
+    }
+    //! Enforce bounds.
+    candidate = arma::min(arma::max(candidate, lowerBound), upperBound);
+}
+
+//! No objectives to evaluate.
+template <typename InitPolicyType, typename DecompPolicyType>
+template<std::size_t I,
+         typename MatType,
+         typename ...ArbitraryFunctionType>
+typename std::enable_if<I == sizeof...(ArbitraryFunctionType), void>::type
+MOEAD<InitPolicyType, DecompPolicyType>::
+EvaluateObjectives(
+    std::vector<MatType>&,
+    std::tuple<ArbitraryFunctionType...>&,
+    std::vector<arma::Col<typename MatType::elem_type> >&)
+{
+ // Nothing to do here.
+}
+
+//! Evaluate the objectives for the entire population.
+template <typename InitPolicyType, typename DecompPolicyType>
+template<std::size_t I,
+         typename MatType,
+         typename ...ArbitraryFunctionType>
+typename std::enable_if<I < sizeof...(ArbitraryFunctionType), void>::type
+MOEAD<InitPolicyType, DecompPolicyType>::
+EvaluateObjectives(
+    std::vector<MatType>& population,
+    std::tuple<ArbitraryFunctionType...>& objectives,
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives)
+{
+  for (size_t i = 0; i < population.size(); i++)
+  {
+    calculatedObjectives[i](I) = std::get<I>(objectives).Evaluate(population[i]);
+    EvaluateObjectives<I+1, MatType, ArbitraryFunctionType...>(population, objectives,
+                                                               calculatedObjectives);
+  }
+}
+
+}  // namespace ens
+
+#endif

include/ensmallen_bits/moead/weight_init_policies/bbs_init.hpp

@@ -0,0 +1,76 @@
+/**
+ * @file bbs_init.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * The Bayesian Bootstrap (BBS) method of Weight Initialization.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOEAD_BBS_HPP
+#define ENSMALLEN_MOEAD_BBS_HPP
+
+namespace ens {
+
+/**
+ * The Bayesian Bootstrap method for initializing weights. Samples are randomly picked from uniform 
+ * distribution followed by sorting and finding adjacent difference. This gives you a list of 
+ * numbers which is guaranteed to sum up to 1.
+ *
+ * @code
+ * @article{rubin1981bayesian,
+ *   title={The bayesian bootstrap},
+ *   author={Rubin, Donald B},
+ *   journal={The annals of statistics},
+ *   pages={130--134},
+ *   year={1981},
+ * @endcode
+ *
+ */
+class BayesianBootstrap
+{
+ public:
+  /**
+   * Constructor for Bayesian Bootstrap policy.
+   */
+  BayesianBootstrap()
+  {
+    /* Nothing to do. */
+  }
+
+  /**
+   * Generate the reference direction matrix.
+   *
+   * @tparam MatType The type of the matrix used for constructing weights.
+   * @param numObjectives The dimensionality of objective space.
+   * @param numPoints The number of reference directions requested.
+   * @param epsilon Handle numerical stability after weight initialization.
+   */
+  template<typename MatType>
+  MatType Generate(const size_t numObjectives,
+                   const size_t numPoints,
+                   const double epsilon)
+  {
+      typedef typename MatType::elem_type ElemType;
+      typedef typename arma::Col<ElemType> VecType;
+
+      MatType weights(numObjectives, numPoints);
+      for (size_t pointIdx = 0; pointIdx < numPoints; ++pointIdx)
+      {
+        VecType referenceDirection(numObjectives + 1, arma::fill::randu);
+        referenceDirection(0) = 0;
+        referenceDirection(numObjectives) = 1;
+        referenceDirection = arma::sort(referenceDirection);
+        referenceDirection = arma::diff(referenceDirection);
+        weights.col(pointIdx) = std::move(referenceDirection) + epsilon;
+      }
+
+      return weights;
+  }
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/moead/weight_init_policies/dirichlet_init.hpp

@@ -0,0 +1,55 @@
+/**
+ * @file dirichlet_init.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * The Dirichlet method of Weight Initialization.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOEAD_DIRICHLET_HPP
+#define ENSMALLEN_MOEAD_DIRICHLET_HPP
+
+namespace ens {
+
+/**
+ * The Dirichlet method for initializing weights. Sampling a 
+ * Dirichlet distribution with parameters set to one returns 
+ * point lying on unit simplex with uniform distribution.
+ */
+class Dirichlet
+{
+ public:
+  /**
+   * Constructor for Dirichlet policy.
+   */
+  Dirichlet()
+  {
+    /* Nothing to do. */
+  }
+
+  /**
+   * Generate the reference direction matrix.
+   *
+   * @tparam MatType The type of the matrix used for constructing weights.
+   * @param numObjectives The dimensionality of objective space.
+   * @param numPoints The number of reference directions requested.
+   * @param epsilon Handle numerical stability after weight initialization.
+   */
+  template<typename MatType>
+  MatType Generate(const size_t numObjectives,
+                   const size_t numPoints,
+                   const double epsilon)
+  {
+    MatType weights = arma::randg<MatType>(numObjectives, numPoints,
+        arma::distr_param(1.0, 1.0)) + epsilon;
+    // Normalize each column.
+    return arma::normalise(weights, 1, 0);
+  }
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/moead/weight_init_policies/uniform_init.hpp

@@ -0,0 +1,208 @@
+/**
+ * @file uniform_init.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * The Uniform (Das Dennis) methodology of Weight Initialization.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOEAD_UNIFORM_HPP
+#define ENSMALLEN_MOEAD_UNIFORM_HPP
+
+namespace ens {
+
+/**
+ * The Uniform (Das Dennis) method for initializing weights. This algorithm guarantees
+ * that the distance between adjacent points would be uniform.
+ *
+ * For more information, see the following:
+ *
+ * @code
+ * article{zhang2007moea,
+ *   title={MOEA/D: A multiobjective evolutionary algorithm based on decomposition},
+ *   author={Zhang, Qingfu and Li, Hui},
+ *   journal={IEEE Transactions on evolutionary computation},
+ *   pages={712--731},
+ *   year={2007}
+ * @endcode
+ */
+class Uniform
+{
+ public:
+  /**
+   * Constructor for Uniform Weight Initializatoin Policy.
+   */
+  Uniform()
+  {
+    /* Nothing to do. */
+  }
+
+  /**
+   * Generate the reference direction matrix.
+   *
+   * @tparam MatType The type of the matrix used for constructing weights.
+   * @param numObjectives The dimensionality of objective space.
+   * @param numPoints The number of reference directions requested.
+   * @param epsilon Handle numerical stability after weight initialization.
+   */
+  template<typename MatType>
+  MatType Generate(size_t numObjectives,
+                   size_t numPoints,
+                   double epsilon)
+  {
+    size_t numPartitions  = FindNumParitions(numObjectives, numPoints);
+    size_t validNumPoints = FindNumUniformPoints(numObjectives, numPartitions);
+
+    //! The requested number of points is not matching any partition number.
+    if (numPoints != validNumPoints)
+    {
+      size_t nextValidNumPoints = FindNumUniformPoints(numObjectives, numPartitions + 1);
+      std::ostringstream oss;
+      oss << "DasDennis::Generate(): " << "The requested numPoints " << numPoints
+          << " cannot be generated uniformly.\n " << "Either choose numPoints as "
+          << validNumPoints << " (numPartition = " << numPartitions << ") or "
+          << "numPoints as " << nextValidNumPoints << " (numPartition = "
+          << numPartitions + 1 << ").";
+      throw std::logic_error(oss.str());
+    }
+
+    return DasDennis<MatType>(numObjectives, numPoints,
+        numPartitions, epsilon);
+  }
+
+ private:
+  /**
+   * Finds the number of points which can be sampled uniformly from a 
+   * unit simplex given the number of partitions.
+   */
+  size_t FindNumUniformPoints(const size_t numObjectives,
+                              const size_t numPartitions)
+  {
+    //! O(N) algorithm to calculate binomial coefficient.
+    //! Source: https://www.geeksforgeeks.org/space-and-time-efficient-binomial-coefficient/
+    auto BinomialCoefficient =
+        [](size_t n, size_t k) -> size_t
+        {
+          size_t retval = 1;
+          // Since, C(n, k) = C(n, n - k).
+          if (k > n - k)
+            k = n - k;
+
+          // [n * (n - 1) * .... * (n - k + 1)] / [k * (k - 1) * .... * 1].
+          for (size_t i = 0; i < k; ++i)
+          {
+            retval *= (n - i);
+            retval /= (i + 1);
+          }
+
+          return retval;
+        };
+    return BinomialCoefficient(numObjectives + numPartitions - 1, numPartitions);
+  }
+
+  /**
+   *  Calculates the appropriate number of partitions such that, the binomial
+   *  coefficient value is closest to the number of points requested.
+   */
+  size_t FindNumParitions(size_t numObjectives, size_t numPoints)
+  {
+    if (numObjectives == 1) return 0;
+    // Iteratively increase numPartitions so that the binomial coefficient
+    // comes near to numPoints;
+    size_t numPartitions {1};
+    size_t sampledNumPoints = FindNumUniformPoints(numPartitions,
+        numObjectives);
+    while (sampledNumPoints <= numPoints)
+    {
+       ++numPartitions;
+       sampledNumPoints = FindNumUniformPoints(numObjectives,
+          numPartitions);
+    }
+
+    return numPartitions - 1;
+  }
+
+  /**
+   * A helper function for DasDennis
+   */
+  template<typename AuxInfoStackType,
+           typename MatType>
+  void DasDennisHelper(AuxInfoStackType& progressStack,
+                       MatType& weights,
+                       const size_t numObjectives,
+                       const size_t numPoints,
+                       const size_t numPartitions,
+                       const double epsilon)
+  {
+    typedef typename MatType::elem_type ElemType;
+    typedef typename arma::Row<ElemType> RowType;
+
+    size_t counter = 0;
+    const ElemType delta = 1.0 / (ElemType)numPartitions;
+
+    while ((counter < numPoints) && !progressStack.empty())
+    {
+      MatType point{};
+      size_t beta{};
+      std::tie(point, beta) = progressStack.back();
+      progressStack.pop_back();
+
+      if (point.size() + 1 == numObjectives)
+      {
+        point.insert_rows(point.n_rows, RowType(1).fill(
+            delta * static_cast<ElemType>(beta)));
+        weights.col(counter) = point + epsilon;
+        ++counter;
+      }
+
+      else
+      {
+        for (size_t i = 0; i <= beta; ++i)
+        {
+          MatType pointClone(point);
+          pointClone.insert_rows(pointClone.n_rows, RowType(1).fill(
+              delta * static_cast<ElemType>(i)));
+          progressStack.push_back({pointClone, beta - i});
+        }
+      }
+    }
+  }
+
+  /**
+   * Generates the weight matrix after verifying the
+   * validity of the parameters.
+   */
+  template <typename MatType>
+  MatType DasDennis(const size_t numObjectives,
+                    const size_t numPoints,
+                    const size_t numPartitions,
+                    const double epsilon)
+  {
+    //! Holds auxillary information required for the helper function. 
+    //! Holds the current point and beta value.
+    using AuxContainer = std::pair<MatType, size_t>;
+
+    std::vector<AuxContainer> progressStack{};
+    //! Init the progress stack.
+    progressStack.push_back({{}, numPartitions});
+    MatType weights(numObjectives, numPoints);
+    weights.fill(arma::datum::nan);
+    DasDennisHelper<decltype(progressStack), MatType>(
+        progressStack,
+        weights,
+        numObjectives,
+        numPoints,
+        numPartitions,
+        epsilon);
+
+    return weights;
+  }
+
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/nsga2/nsga2.hpp

@@ -0,0 +1,392 @@
+/**
+ * @file nsga2.hpp
+ * @author Sayan Goswami
+ * @author Nanubala Gnana Sai
+ *
+ * NSGA-II is a multi-objective optimization algorithm, widely used in
+ * many real-world applications. NSGA-II generates offsprings using
+ * crossover and mutation and then selects the next generation according
+ * to non-dominated-sorting and crowding distance comparison.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_NSGA2_NSGA2_HPP
+#define ENSMALLEN_NSGA2_NSGA2_HPP
+
+namespace ens {
+
+/**
+ * NSGA-II (Non-dominated Sorting Genetic Algorithm - II) is a multi-objective
+ * optimization algorithm. This class implements the NSGA-II algorithm.
+ *
+ * The algorithm works by generating a candidate population from a fixed
+ * starting point. At each stage of optimization, a new population of children
+ * is generated. This new population along with its predecessor is sorted using
+ * non-domination as the metric. Following this, the population is further
+ * segregated in fronts. A new population is generated from these fronts having
+ * size equal to that of the starting population.
+ *
+ * During evolution, two parents are randomly chosen using binary tournament
+ * selection. A pair of children are generated by crossing over these two
+ * candidates followed by mutation.
+ *
+ * The best front (Pareto optimal) is returned by the Optimize() method.
+ *
+ * For more information, see the following:
+ *
+ * @code
+ * @article{10.1109/4235.996017,
+ *   author = {Deb, K. and Pratap, A. and Agarwal, S. and Meyarivan, T.},
+ *   title = {A Fast and Elitist Multiobjective Genetic Algorithm: NSGA-II},
+ *   year = {2002},
+ *   url = {https://doi.org/10.1109/4235.996017},
+ *   journal = {Trans. Evol. Comp}}
+ * @endcode
+ *
+ * NSGA-II can optimize arbitrary multi-objective functions. For more details,
+ * see the documentation on function types included with this distribution or
+ * on the ensmallen website.
+ */
+class NSGA2
+{
+ public:
+  /**
+   * Constructor for the NSGA2 optimizer.
+   *
+   * The default values provided over here are not necessarily suitable for a
+   * given function. Therefore it is highly recommended to adjust the
+   * parameters according to the problem.
+   *
+   * @param populationSize The number of candidates in the population.
+   *     This should be atleast 4 in size and a multiple of 4.
+   * @param maxGenerations The maximum number of generations allowed for NSGA-II.
+   * @param crossoverProb The probability that a crossover will occur.
+   * @param mutationProb The probability that a mutation will occur.
+   * @param mutationStrength The strength of the mutation.
+   * @param epsilon The minimum difference required to distinguish between
+   *     candidate solutions.
+   * @param lowerBound Lower bound of the coordinates of the initial population.
+   * @param upperBound Upper bound of the coordinates of the initial population.
+   */
+  NSGA2(const size_t populationSize = 100,
+        const size_t maxGenerations = 2000,
+        const double crossoverProb = 0.6,
+        const double mutationProb = 0.3,
+        const double mutationStrength = 1e-3,
+        const double epsilon = 1e-6,
+        const arma::vec& lowerBound = arma::zeros(1, 1),
+        const arma::vec& upperBound = arma::ones(1, 1));
+
+  /**
+   * Constructor for the NSGA2 optimizer. This constructor provides an overload
+   * to use `lowerBound` and `upperBound` of type double.
+   *
+   * The default values provided over here are not necessarily suitable for a
+   * given function. Therefore it is highly recommended to adjust the
+   * parameters according to the problem.
+   *
+   * @param populationSize The number of candidates in the population.
+   *     This should be atleast 4 in size and a multiple of 4.
+   * @param maxGenerations The maximum number of generations allowed for NSGA-II.
+   * @param crossoverProb The probability that a crossover will occur.
+   * @param mutationProb The probability that a mutation will occur.
+   * @param mutationStrength The strength of the mutation.
+   * @param epsilon The minimum difference required to distinguish between
+   *     candidate solutions.
+   * @param lowerBound Lower bound of the coordinates of the initial population.
+   * @param upperBound Upper bound of the coordinates of the initial population.
+   */
+  NSGA2(const size_t populationSize = 100,
+        const size_t maxGenerations = 2000,
+        const double crossoverProb = 0.6,
+        const double mutationProb = 0.3,
+        const double mutationStrength = 1e-3,
+        const double epsilon = 1e-6,
+        const double lowerBound = 0,
+        const double upperBound = 1);
+
+  /**
+   * Optimize a set of objectives. The initial population is generated using the
+   * starting point. The output is the best generated front.
+   *
+   * @tparam ArbitraryFunctionType std::tuple of multiple objectives.
+   * @tparam MatType Type of matrix to optimize.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param objectives Vector of objective functions to optimize for.
+   * @param iterate Starting point.
+   * @param callbacks Callback functions.
+   * @return MatType::elem_type The minimum of the accumulated sum over the
+   *     objective values in the best front.
+   */
+  template<typename MatType,
+           typename... ArbitraryFunctionType,
+           typename... CallbackTypes>
+ typename MatType::elem_type Optimize(
+     std::tuple<ArbitraryFunctionType...>& objectives,
+     MatType& iterate,
+     CallbackTypes&&... callbacks);
+
+  //! Get the population size.
+  size_t PopulationSize() const { return populationSize; }
+  //! Modify the population size.
+  size_t& PopulationSize() { return populationSize; }
+
+  //! Get the maximum number of generations.
+  size_t MaxGenerations() const { return maxGenerations; }
+  //! Modify the maximum number of generations.
+  size_t& MaxGenerations() { return maxGenerations; }
+
+  //! Get the crossover rate.
+  double CrossoverRate() const { return crossoverProb; }
+  //! Modify the crossover rate.
+  double& CrossoverRate() { return crossoverProb; }
+
+  //! Get the mutation probability.
+  double MutationProbability() const { return mutationProb; }
+  //! Modify the mutation probability.
+  double& MutationProbability() { return mutationProb; }
+
+  //! Get the mutation strength.
+  double MutationStrength() const { return mutationStrength; }
+  //! Modify the mutation strength.
+  double& MutationStrength() { return mutationStrength; }
+
+  //! Get the tolerance.
+  double Epsilon() const { return epsilon; }
+  //! Modify the tolerance.
+  double& Epsilon() { return epsilon; }
+
+  //! Retrieve value of lowerBound.
+  const arma::vec& LowerBound() const { return lowerBound; }
+  //! Modify value of lowerBound.
+  arma::vec& LowerBound() { return lowerBound; }
+
+  //! Retrieve value of upperBound.
+  const arma::vec& UpperBound() const { return upperBound; }
+  //! Modify value of upperBound.
+  arma::vec& UpperBound() { return upperBound; }
+
+  //! Retrieve the Pareto optimal points in variable space. This returns an empty cube
+  //! until `Optimize()` has been called.
+  const arma::cube& ParetoSet() const { return paretoSet; }
+
+  //! Retrieve the best front (the Pareto frontier). This returns an empty cube until
+  //! `Optimize()` has been called.
+  const arma::cube& ParetoFront() const { return paretoFront; }
+
+  /**
+   * Retrieve the best front (the Pareto frontier).  This returns an empty
+   * vector until `Optimize()` has been called.  Note that this function is
+   * deprecated and will be removed in ensmallen 3.x!  Use `ParetoFront()`
+   * instead.
+   */
+  ens_deprecated const std::vector<arma::mat>& Front()
+  {
+    if (rcFront.size() == 0)
+    {
+      // Match the old return format.
+      for (size_t i = 0; i < paretoFront.n_slices; ++i)
+      {
+        rcFront.push_back(arma::mat(paretoFront.slice(i)));
+      }
+    }
+
+    return rcFront;
+  }
+
+ private:
+  /**
+   * Evaluate objectives for the elite population.
+   *
+   * @tparam ArbitraryFunctionType std::tuple of multiple function types.
+   * @tparam MatType Type of matrix to optimize.
+   * @param population The elite population.
+   * @param objectives The set of objectives.
+   * @param calculatedObjectives Vector to store calculated objectives.
+   */
+  template<std::size_t I = 0,
+           typename MatType,
+           typename ...ArbitraryFunctionType>
+  typename std::enable_if<I == sizeof...(ArbitraryFunctionType), void>::type
+  EvaluateObjectives(std::vector<MatType>&,
+                     std::tuple<ArbitraryFunctionType...>&,
+                     std::vector<arma::Col<typename MatType::elem_type> >&);
+
+  template<std::size_t I = 0,
+           typename MatType,
+           typename ...ArbitraryFunctionType>
+  typename std::enable_if<I < sizeof...(ArbitraryFunctionType), void>::type
+  EvaluateObjectives(std::vector<MatType>& population,
+                     std::tuple<ArbitraryFunctionType...>& objectives,
+                     std::vector<arma::Col<typename MatType::elem_type> >&
+                     calculatedObjectives);
+
+  /**
+   * Reproduce candidates from the elite population to generate a new
+   * population.
+   *
+   * @tparam MatType Type of matrix to optimize.
+   * @param population The elite population.
+   * @param objectives The set of objectives.
+   * @param lowerBound Lower bound of the coordinates of the initial population.
+   * @param upperBound Upper bound of the coordinates of the initial population.
+   */
+  template<typename MatType>
+  void BinaryTournamentSelection(std::vector<MatType>& population,
+                                 const MatType& lowerBound,
+                                 const MatType& upperBound);
+
+  /**
+   * Crossover two parents to create a pair of new children.
+   *
+   * @tparam MatType Type of matrix to optimize.
+   * @param childA A newly generated candidate.
+   * @param childB Another newly generated candidate.
+   * @param parentA First parent from elite population.
+   * @param parentB Second parent from elite population.
+   */
+  template<typename MatType>
+  void Crossover(MatType& childA,
+                 MatType& childB,
+                 const MatType& parentA,
+                 const MatType& parentB);
+
+  /**
+   * Mutate the coordinates for a candidate.
+   *
+   * @tparam MatType Type of matrix to optimize.
+   * @param child The candidate whose coordinates are being modified.
+   * @param objectives The set of objectives.
+   * @param lowerBound Lower bound of the coordinates of the initial population.
+   * @param upperBound Upper bound of the coordinates of the initial population.
+   */
+  template<typename MatType>
+  void Mutate(MatType& child,
+              const MatType& lowerBound,
+              const MatType& upperBound);
+
+  /**
+   * Sort the candidate population using their domination count and the set of
+   * dominated nodes.
+   *
+   * @tparam MatType Type of matrix to optimize.
+   * @param fronts The population is sorted into these Pareto fronts. The first
+   *     front is the best, the second worse and so on.
+   * @param ranks The assigned ranks, used for crowding distance based sorting.
+   * @param calculatedObjectives The previously calculated objectives.
+   */
+  template<typename MatType>
+  void FastNonDominatedSort(
+      std::vector<std::vector<size_t> >& fronts,
+      std::vector<size_t>& ranks,
+      std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives);
+
+  /**
+   * Operator to check if one candidate Pareto-dominates the other.
+   *
+   * A candidate is said to dominate the other if it is at least as good as the
+   * other candidate for all the objectives and there exists at least one
+   * objective for which it is strictly better than the other candidate.
+   *
+   * @tparam MatType Type of matrix to optimize.
+   * @param calculatedObjectives The previously calculated objectives.
+   * @param candidateP The candidate being compared from the elite population.
+   * @param candidateQ The candidate being compared against.
+   * @return true if candidateP Pareto dominates candidateQ, otherwise, false.
+   */
+  template<typename MatType>
+  bool Dominates(
+      std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+      size_t candidateP,
+      size_t candidateQ);
+
+  /**
+   * Assigns crowding distance metric for sorting.
+   *
+   * @param front The previously generated Pareto fronts.
+   * @param calculatedObjectives The previously calculated objectives.
+   * @param crowdingDistance The crowding distance for each individual in
+   *    the population.
+   */
+  template <typename MatType>
+  void CrowdingDistanceAssignment(
+      const std::vector<size_t>& front,
+      std::vector<arma::Col<typename MatType::elem_type>>& calculatedObjectives,
+      std::vector<typename MatType::elem_type>& crowdingDistance);
+
+  /**
+   * The operator used in the crowding distance based sorting.
+   *
+   * If a candidates has a lower rank then it is preferred.
+   * Otherwise, if the ranks are equal then the candidate with the larger
+   * crowding distance is preferred.
+   *
+   * @param idxP The index of the first cadidate from the elite population being
+   *     sorted.
+   * @param idxQ The index of the second cadidate from the elite population
+   *     being sorted.
+   * @param ranks The previously calculated ranks.
+   * @param crowdingDistance The crowding distance for each individual in
+   *    the population.
+   * @return true if the first candidate is preferred, otherwise, false.
+   */
+  template<typename MatType>
+  bool CrowdingOperator(size_t idxP,
+                        size_t idxQ,
+                        const std::vector<size_t>& ranks,
+                        const std::vector<typename MatType::elem_type>& crowdingDistance);
+
+  //! The number of objectives being optimised for.
+  size_t numObjectives;
+
+  //! The numbeer of variables used per objectives.
+  size_t numVariables;
+
+  //! The number of candidates in the population.
+  size_t populationSize;
+
+  //! Maximum number of generations before termination criteria is met.
+  size_t maxGenerations;
+
+  //! Probability that crossover will occur.
+  double crossoverProb;
+
+  //! Probability that mutation will occur.
+  double mutationProb;
+
+  //! Strength of the mutation.
+  double mutationStrength;
+
+  //! The tolerance for termination.
+  double epsilon;
+
+  //! Lower bound of the initial swarm.
+  arma::vec lowerBound;
+
+  //! Upper bound of the initial swarm.
+  arma::vec upperBound;
+
+  //! The set of all the Pareto optimal points.
+  //! Stored after Optimize() is called.
+  arma::cube paretoSet;
+
+  //! The set of all the Pareto optimal objective vectors.
+  //! Stored after Optimize() is called.
+  arma::cube paretoFront;
+
+  //! A different representation of the Pareto front, for reverse compatibility
+  //! purposes.  This can be removed when ensmallen 3.x is released!  (Along
+  //! with `Front()`.)  This is only populated when `Front()` is called.
+  std::vector<arma::mat> rcFront;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "nsga2_impl.hpp"
+
+#endif

include/ensmallen_bits/nsga2/nsga2_impl.hpp

@@ -0,0 +1,488 @@
+/**
+ * @file nsga2_impl.hpp
+ * @author Sayan Goswami
+ * @author Nanubala Gnana Sai
+ *
+ * Implementation of the NSGA-II algorithm. Used for multi-objective
+ * optimization problems on arbitrary functions.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more Information.
+ */
+
+#ifndef ENSMALLEN_NSGA2_NSGA2_IMPL_HPP
+#define ENSMALLEN_NSGA2_NSGA2_IMPL_HPP
+
+#include "nsga2.hpp"
+#include <assert.h>
+
+namespace ens {
+
+inline NSGA2::NSGA2(const size_t populationSize,
+                    const size_t maxGenerations,
+                    const double crossoverProb,
+                    const double mutationProb,
+                    const double mutationStrength,
+                    const double epsilon,
+                    const arma::vec& lowerBound,
+                    const arma::vec& upperBound) :
+    numObjectives(0),
+    numVariables(0),
+    populationSize(populationSize),
+    maxGenerations(maxGenerations),
+    crossoverProb(crossoverProb),
+    mutationProb(mutationProb),
+    mutationStrength(mutationStrength),
+    epsilon(epsilon),
+    lowerBound(lowerBound),
+    upperBound(upperBound)
+{ /* Nothing to do here. */ }
+
+inline NSGA2::NSGA2(const size_t populationSize,
+                    const size_t maxGenerations,
+                    const double crossoverProb,
+                    const double mutationProb,
+                    const double mutationStrength,
+                    const double epsilon,
+                    const double lowerBound,
+                    const double upperBound) :
+    numObjectives(0),
+    numVariables(0),
+    populationSize(populationSize),
+    maxGenerations(maxGenerations),
+    crossoverProb(crossoverProb),
+    mutationProb(mutationProb),
+    mutationStrength(mutationStrength),
+    epsilon(epsilon),
+    lowerBound(lowerBound * arma::ones(1, 1)),
+    upperBound(upperBound * arma::ones(1, 1))
+{ /* Nothing to do here. */ }
+
+//! Optimize the function.
+template<typename MatType,
+         typename... ArbitraryFunctionType,
+         typename... CallbackTypes>
+typename MatType::elem_type NSGA2::Optimize(
+    std::tuple<ArbitraryFunctionType...>& objectives,
+    MatType& iterateIn,
+    CallbackTypes&&... callbacks)
+{
+  // Make sure for evolution to work at least four candidates are present.
+  if (populationSize < 4 && populationSize % 4 != 0)
+  {
+    throw std::logic_error("NSGA2::Optimize(): population size should be at"
+        " least 4, and, a multiple of 4!");
+  }
+
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  // Make sure that we have the methods that we need.  Long name...
+  traits::CheckArbitraryFunctionTypeAPI<ArbitraryFunctionType...,
+      BaseMatType>();
+  RequireDenseFloatingPointType<BaseMatType>();
+
+  // Check if lower bound is a vector of a single dimension.
+  if (lowerBound.n_rows == 1)
+    lowerBound = lowerBound(0, 0) * arma::ones(iterate.n_rows, iterate.n_cols);
+
+  // Check if upper bound is a vector of a single dimension.
+  if (upperBound.n_rows == 1)
+    upperBound = upperBound(0, 0) * arma::ones(iterate.n_rows, iterate.n_cols);
+
+  // Check the dimensions of lowerBound and upperBound.
+  assert(lowerBound.n_rows == iterate.n_rows && "The dimensions of "
+      "lowerBound are not the same as the dimensions of iterate.");
+  assert(upperBound.n_rows == iterate.n_rows && "The dimensions of "
+      "upperBound are not the same as the dimensions of iterate.");
+
+  numObjectives = sizeof...(ArbitraryFunctionType);
+  numVariables = iterate.n_rows;
+
+  // Cache calculated objectives.
+  std::vector<arma::Col<ElemType> > calculatedObjectives(populationSize);
+
+  // Population size reserved to 2 * populationSize + 1 to accommodate
+  // for the size of intermediate candidate population.
+  std::vector<BaseMatType> population;
+  population.reserve(2 * populationSize + 1);
+
+  // Pareto fronts, initialized during non-dominated sorting.
+  // Stores indices of population belonging to a certain front.
+  std::vector<std::vector<size_t> > fronts;
+  // Initialised in CrowdingDistanceAssignment.
+  std::vector<ElemType> crowdingDistance;
+  // Initialised during non-dominated sorting.
+  std::vector<size_t> ranks;
+
+  //! Useful temporaries for float-like comparisons.
+  const BaseMatType castedLowerBound = arma::conv_to<BaseMatType>::from(lowerBound);
+  const BaseMatType castedUpperBound = arma::conv_to<BaseMatType>::from(upperBound);
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
+  // Generate the population based on a uniform distribution around the given
+  // starting point.
+  for (size_t i = 0; i < populationSize; i++)
+  {
+    population.push_back(arma::randu<BaseMatType>(iterate.n_rows,
+        iterate.n_cols) - 0.5 + iterate);
+
+    // Constrain all genes to be within bounds.
+    population[i] = arma::min(arma::max(population[i], castedLowerBound), castedUpperBound);
+  }
+
+  Info << "NSGA2 initialized successfully. Optimization started." << std::endl;
+
+  // Iterate until maximum number of generations is obtained.
+  terminate |= Callback::BeginOptimization(*this, objectives, iterate, callbacks...);
+
+  for (size_t generation = 1; generation <= maxGenerations && !terminate; generation++)
+  {
+    Info << "NSGA2: iteration " << generation << "." << std::endl;
+
+    // Create new population of candidate from the present elite population.
+    // Have P_t, generate G_t using P_t.
+    BinaryTournamentSelection(population, castedLowerBound, castedUpperBound);
+
+    // Evaluate the objectives for the new population.
+    calculatedObjectives.resize(population.size());
+    std::fill(calculatedObjectives.begin(), calculatedObjectives.end(),
+        arma::Col<ElemType>(numObjectives, arma::fill::zeros));
+    EvaluateObjectives(population, objectives, calculatedObjectives);
+
+    // Perform fast non dominated sort on P_t ∪ G_t.
+    ranks.resize(population.size());
+    FastNonDominatedSort<BaseMatType>(fronts, ranks, calculatedObjectives);
+
+    // Perform crowding distance assignment.
+    crowdingDistance.resize(population.size());
+    std::fill(crowdingDistance.begin(), crowdingDistance.end(), 0.);
+    for (size_t fNum = 0; fNum < fronts.size(); fNum++)
+    {
+      CrowdingDistanceAssignment<BaseMatType>(
+          fronts[fNum], calculatedObjectives, crowdingDistance);
+    }
+
+    // Sort based on crowding distance.
+    std::sort(population.begin(), population.end(),
+      [this, ranks, crowdingDistance, population]
+        (BaseMatType candidateP, BaseMatType candidateQ)
+          {
+            size_t idxP{}, idxQ{};
+            for (size_t i = 0; i < population.size(); i++)
+            {
+              if (arma::approx_equal(population[i], candidateP, "absdiff", epsilon))
+                idxP = i;
+
+              if (arma::approx_equal(population[i], candidateQ, "absdiff", epsilon))
+                idxQ = i;
+            }
+
+            return CrowdingOperator<BaseMatType>(idxP, idxQ, ranks, crowdingDistance);
+          }
+    );
+
+    // Yield a new population P_{t+1} of size populationSize.
+    // Discards unfit population from the R_{t} to yield P_{t+1}.
+    population.resize(populationSize);
+
+    terminate |= Callback::GenerationalStepTaken(*this, objectives, iterate,
+        calculatedObjectives, fronts, callbacks...);
+  }
+
+  // Set the candidates from the Pareto Set as the output.
+  paretoSet.set_size(population[0].n_rows, population[0].n_cols, fronts[0].size());
+  // The Pareto Set is stored, can be obtained via ParetoSet() getter.
+  for (size_t solutionIdx = 0; solutionIdx < fronts[0].size(); ++solutionIdx)
+  {
+    paretoSet.slice(solutionIdx) =
+      arma::conv_to<arma::mat>::from(population[fronts[0][solutionIdx]]);
+  }
+
+  // Set the candidates from the Pareto Front as the output.
+  paretoFront.set_size(calculatedObjectives[0].n_rows, calculatedObjectives[0].n_cols,
+      fronts[0].size());
+  // The Pareto Front is stored, can be obtained via ParetoFront() getter.
+  for (size_t solutionIdx = 0; solutionIdx < fronts[0].size(); ++solutionIdx)
+  {
+    paretoFront.slice(solutionIdx) =
+      arma::conv_to<arma::mat>::from(calculatedObjectives[fronts[0][solutionIdx]]);
+  }
+
+  // Clear rcFront, in case it is later requested by the user for reverse
+  // compatibility reasons.
+  rcFront.clear();
+
+  // Assign iterate to first element of the Pareto Set.
+  iterate = population[fronts[0][0]];
+
+  Callback::EndOptimization(*this, objectives, iterate, callbacks...);
+
+  ElemType performance = std::numeric_limits<ElemType>::max();
+
+  for (const arma::Col<ElemType>& objective: calculatedObjectives)
+    if (arma::accu(objective) < performance)
+      performance = arma::accu(objective);
+
+  return performance;
+}
+
+//! No objectives to evaluate.
+template<std::size_t I,
+         typename MatType,
+         typename ...ArbitraryFunctionType>
+typename std::enable_if<I == sizeof...(ArbitraryFunctionType), void>::type
+NSGA2::EvaluateObjectives(
+    std::vector<MatType>&,
+    std::tuple<ArbitraryFunctionType...>&,
+    std::vector<arma::Col<typename MatType::elem_type> >&)
+{
+  // Nothing to do here.
+}
+
+//! Evaluate the objectives for the entire population.
+template<std::size_t I,
+         typename MatType,
+         typename ...ArbitraryFunctionType>
+typename std::enable_if<I < sizeof...(ArbitraryFunctionType), void>::type
+NSGA2::EvaluateObjectives(
+    std::vector<MatType>& population,
+    std::tuple<ArbitraryFunctionType...>& objectives,
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives)
+{
+  for (size_t i = 0; i < populationSize; i++)
+  {
+    calculatedObjectives[i](I) = std::get<I>(objectives).Evaluate(population[i]);
+    EvaluateObjectives<I+1, MatType, ArbitraryFunctionType...>(population, objectives,
+                                                               calculatedObjectives);
+  }
+}
+
+//! Reproduce and generate new candidates.
+template<typename MatType>
+inline void NSGA2::BinaryTournamentSelection(std::vector<MatType>& population,
+                                             const MatType& lowerBound,
+                                             const MatType& upperBound)
+{
+  std::vector<MatType> children;
+
+  while (children.size() < population.size())
+  {
+    // Choose two random parents for reproduction from the elite population.
+    size_t indexA = arma::randi<size_t>(arma::distr_param(0, populationSize - 1));
+    size_t indexB = arma::randi<size_t>(arma::distr_param(0, populationSize - 1));
+
+    // Make sure that the parents differ.
+    if (indexA == indexB)
+    {
+      if (indexB < populationSize - 1)
+        indexB++;
+      else
+        indexB--;
+    }
+
+    // Initialize the children to the respective parents.
+    MatType childA = population[indexA], childB = population[indexB];
+
+    Crossover(childA, childB, population[indexA], population[indexB]);
+
+    Mutate(childA, lowerBound, upperBound);
+    Mutate(childB, lowerBound, upperBound);
+
+    // Add the children to the candidate population.
+    children.push_back(childA);
+    children.push_back(childB);
+  }
+
+  // Add the candidates to the elite population.
+  population.insert(std::end(population), std::begin(children), std::end(children));
+}
+
+//! Perform crossover of genes for the children.
+template<typename MatType>
+inline void NSGA2::Crossover(MatType& childA,
+                             MatType& childB,
+                             const MatType& parentA,
+                             const MatType& parentB)
+{
+  // Indices at which crossover is to occur.
+  const arma::umat idx = arma::randu<MatType>(childA.n_rows, childA.n_cols) < crossoverProb;
+
+  // Use traits from parentA for indices where idx is 1 and parentB otherwise.
+  childA = parentA % idx + parentB % (1 - idx);
+  // Use traits from parentB for indices where idx is 1 and parentA otherwise.
+  childB = parentA % (1 - idx) + parentA % idx;
+}
+
+//! Perform mutation of the candidates weights with some noise.
+template<typename MatType>
+inline void NSGA2::Mutate(MatType& child,
+                          const MatType& lowerBound,
+                          const MatType& upperBound)
+{
+  child += (arma::randu<MatType>(child.n_rows, child.n_cols) < mutationProb) %
+      (mutationStrength * arma::randn<MatType>(child.n_rows, child.n_cols));
+
+  // Constrain all genes to be between bounds.
+  child = arma::min(arma::max(child, lowerBound), upperBound);
+}
+
+//! Sort population into Pareto fronts.
+template<typename MatType>
+inline void NSGA2::FastNonDominatedSort(
+    std::vector<std::vector<size_t> >& fronts,
+    std::vector<size_t>& ranks,
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives)
+{
+  std::map<size_t, size_t> dominationCount;
+  std::map<size_t, std::set<size_t> > dominated;
+
+  // Reset and initialize fronts.
+  fronts.clear();
+  fronts.push_back(std::vector<size_t>());
+
+  for (size_t p = 0; p < populationSize; p++)
+  {
+    dominated[p] = std::set<size_t>();
+    dominationCount[p] = 0;
+
+    for (size_t q = 0; q < populationSize; q++)
+    {
+      if (Dominates<MatType>(calculatedObjectives, p, q))
+        dominated[p].insert(q);
+      else if (Dominates<MatType>(calculatedObjectives, q, p))
+        dominationCount[p] += 1;
+    }
+
+    if (dominationCount[p] == 0)
+    {
+      ranks[p] = 0;
+      fronts[0].push_back(p);
+    }
+  }
+
+  size_t i = 0;
+
+  while (!fronts[i].empty())
+  {
+    std::vector<size_t> nextFront;
+
+    for (size_t p: fronts[i])
+    {
+      for (size_t q: dominated[p])
+      {
+        dominationCount[q]--;
+
+        if (dominationCount[q] == 0)
+        {
+          ranks[q] = i + 1;
+          nextFront.push_back(q);
+        }
+      }
+    }
+
+    i++;
+    fronts.push_back(nextFront);
+  }
+  // Remove the empty final set.
+  fronts.pop_back();
+}
+
+//! Check if a candidate Pareto dominates another candidate.
+template<typename MatType>
+inline bool NSGA2::Dominates(
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+    size_t candidateP,
+    size_t candidateQ)
+{
+  bool allBetterOrEqual = true;
+  bool atleastOneBetter = false;
+  size_t n_objectives = calculatedObjectives[0].n_elem;
+
+  for (size_t i = 0; i < n_objectives; i++)
+  {
+    // P is worse than Q for the i-th objective function.
+    if (calculatedObjectives[candidateP](i) > calculatedObjectives[candidateQ](i))
+      allBetterOrEqual = false;
+
+    // P is better than Q for the i-th objective function.
+    else if (calculatedObjectives[candidateP](i) < calculatedObjectives[candidateQ](i))
+      atleastOneBetter = true;
+  }
+
+  return allBetterOrEqual && atleastOneBetter;
+}
+
+//! Assign crowding distance to the population.
+template <typename MatType>
+inline void NSGA2::CrowdingDistanceAssignment(
+	const std::vector<size_t>& front,
+	std::vector<arma::Col<typename MatType::elem_type>>& calculatedObjectives,
+	std::vector<typename MatType::elem_type>& crowdingDistance)
+{
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+
+  size_t fSize = front.size();
+  // Stores the sorted indices of the fronts.
+  arma::uvec sortedIdx  = arma::regspace<arma::uvec>(0, 1, fSize - 1);
+
+  for (size_t m = 0; m < numObjectives; m++)
+  {
+    // Cache fValues of individuals for current objective.
+    arma::Col<ElemType> fValues(fSize);
+    std::transform(front.begin(), front.end(), fValues.begin(),
+      [&](const size_t& individual)
+        {
+          return calculatedObjectives[individual](m);
+        });
+
+    // Sort front indices by ascending fValues for current objective.
+    std::sort(sortedIdx.begin(), sortedIdx.end(),
+      [&](const size_t& frontIdxA, const size_t& frontIdxB)
+        {
+          return (fValues(frontIdxA) < fValues(frontIdxB));
+        });
+
+    crowdingDistance[front[sortedIdx(0)]] =
+        std::numeric_limits<ElemType>::max();
+    crowdingDistance[front[sortedIdx(fSize - 1)]] =
+        std::numeric_limits<ElemType>::max();
+    ElemType minFval = fValues(sortedIdx(0));
+    ElemType maxFval = fValues(sortedIdx(fSize - 1));
+    ElemType scale =
+        std::abs(maxFval - minFval) == 0. ? 1. : std::abs(maxFval - minFval);
+
+    for (size_t i = 1; i < fSize - 1; i++)
+    {
+      crowdingDistance[front[sortedIdx(i)]] +=
+          (fValues(sortedIdx(i + 1)) - fValues(sortedIdx(i - 1))) / scale;
+    }
+  }
+}
+
+//! Comparator for crowding distance based sorting.
+template<typename MatType>
+inline bool NSGA2::CrowdingOperator(size_t idxP,
+                                    size_t idxQ,
+                                    const std::vector<size_t>& ranks,
+                                    const std::vector<typename MatType::elem_type>& crowdingDistance)
+{
+  if (ranks[idxP] < ranks[idxQ])
+    return true;
+  else if (ranks[idxP] == ranks[idxQ] && crowdingDistance[idxP] > crowdingDistance[idxQ])
+    return true;
+
+  return false;
+}
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/padam/padam.hpp

@@ -113,7 +113,7 @@ class Padam
   {
     return optimizer.template Optimize<
         SeparableFunctionType, MatType, GradType, CallbackTypes...>(
-        function, iterate, callbacks...);
+        function, iterate, std::forward<CallbackTypes>(callbacks)...);
   }
 
   //! Forward the MatType as GradType.

include/ensmallen_bits/parallel_sgd/parallel_sgd.hpp

@@ -22,6 +22,8 @@ namespace ens {
  * HOGWILD! approach.
  *
  * For more information, see the following.
+ *
+ * @code
  * @misc{1106.5730,
  *   Author = {Feng Niu and Benjamin Recht and Christopher Re and Stephen J.
  *             Wright},
@@ -30,6 +32,7 @@ namespace ens {
  *   Year   = {2011},
  *   Eprint = {arXiv:1106.5730},
  * }
+ * @endcode
  *
  * ParallelSGD can optimize sparse differentiable separable functions.  For more
  * details, see the documentation on function types included with this

include/ensmallen_bits/problems/ackley_function.hpp

@@ -59,10 +59,6 @@ class AckleyFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const { return MatType("-5.0; 5.0"); }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -116,6 +112,22 @@ class AckleyFunction
   //! Modify the value used for numerical stability.
   double& Epsilon() { return epsilon; }
 
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("0.02; 0.02"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("0.0; 0.0"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 0.0; }
+
  private:
   //! The value of the multiplicative constant.
   double c;

include/ensmallen_bits/problems/aug_lagrangian_test_functions_impl.hpp

@@ -129,14 +129,15 @@ inline typename MatType::elem_type GockenbachFunction::EvaluateConstraint(
   switch (index)
   {
     case 0: // g(x) = (x_3 - x_2 - x_1 - 1) = 0
-      constraint = (coordinates[2] - coordinates[1] - coordinates[0] - 1);
+      constraint = (coordinates[2] - coordinates[1] - coordinates[0] -
+          typename MatType::elem_type(1));
       break;
 
     case 1: // h(x) = (x_3 - x_1^2) >= 0
       // To deal with the inequality, the constraint will simply evaluate to 0
       // when h(x) >= 0.
-      constraint = std::min(0.0,
-          (coordinates[2] - std::pow(coordinates[0], 2)));
+      constraint = std::min(typename MatType::elem_type(0), (coordinates[2] -
+          std::pow(coordinates[0], typename MatType::elem_type(2))));
       break;
   }
 

include/ensmallen_bits/problems/beale_function.hpp

@@ -52,10 +52,6 @@ class BealeFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const { return MatType("-4.5; 4.5"); }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -98,6 +94,22 @@ class BealeFunction
    */
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient);
+
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("2.8; 0.35"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("3.0; 0.5"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 0.0; }
 };
 
 } // namespace test

include/ensmallen_bits/problems/booth_function.hpp

@@ -52,10 +52,6 @@ class BoothFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const { return MatType("-9; -9"); }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -98,6 +94,22 @@ class BoothFunction
    */
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient);
+
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("-9; -9"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("1.0; 3.0"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 0.0; }
 };
 
 } // namespace test

include/ensmallen_bits/problems/bukin_function.hpp

@@ -57,10 +57,6 @@ class BukinFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const { return MatType("-10; -2.0"); }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -109,6 +105,22 @@ class BukinFunction
   //! Modify the value used for numerical stability.
   double& Epsilon() { return epsilon; }
 
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("-10; -2.0"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("-10.0; 1.0"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 0.0; }
+
  private:
   //! The value used for numerical stability.
   double epsilon;

include/ensmallen_bits/problems/colville_function.hpp

@@ -53,10 +53,6 @@ class ColvilleFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const { return MatType("-5; 3; 1; -9"); }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -99,6 +95,22 @@ class ColvilleFunction
    */
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient) const;
+
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("-5; 3; 1; -9"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("1; 1; 1; 1"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 0.0; }
 };
 
 } // namespace test

include/ensmallen_bits/problems/cross_in_tray_function.hpp

@@ -57,10 +57,6 @@ class CrossInTrayFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const { return MatType("0; 0"); }
-
   /*
    * Evaluate a function for a particular batch-size.
    *
@@ -80,6 +76,14 @@ class CrossInTrayFunction
    */
   template<typename MatType>
   typename MatType::elem_type Evaluate(const MatType& coordinates) const;
+
+  // Note: GetInitialPoint() is not required for using ensmallen to optimize
+  // this function!  It is specifically used as a convenience just for
+  // ensmallen's testing infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("0; 0"); }
 };
 
 } // namespace test

include/ensmallen_bits/problems/drop_wave_function.hpp

@@ -52,10 +52,6 @@ class DropWaveFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const { return MatType("0.5; 0.5"); }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -98,6 +94,22 @@ class DropWaveFunction
    */
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient);
+
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("0.5; 0.5"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("0.0; 0.0"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 0.0; }
 };
 
 } // namespace test

include/ensmallen_bits/problems/easom_function.hpp

@@ -52,10 +52,6 @@ class EasomFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const { return MatType("-90.0; 90.0"); }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -98,6 +94,22 @@ class EasomFunction
    */
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient);
+
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("2.9; 2.9"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("3.14; 3.14"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return -1.0; }
 };
 
 } // namespace test

include/ensmallen_bits/problems/eggholder_function.hpp

@@ -53,10 +53,6 @@ class EggholderFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const { return MatType("-333; -333"); }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -99,6 +95,22 @@ class EggholderFunction
    */
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient);
+
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("-333; -333"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("512; 404.2319"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return -959.6407; }
 };
 
 } // namespace test

include/ensmallen_bits/problems/fonseca_fleming_function.hpp

@@ -0,0 +1,106 @@
+/**
+ * @file fonseca_fleming_function_n1.hpp
+ * @author Sayan Goswami
+ *
+ * Implementation of Fonseca Fleming function.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_FONSECA_FLEEMING_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_FONSECA_FLEEMING_FUNCTION_HPP
+
+#include <tuple>
+
+namespace ens {
+namespace test {
+
+/**
+ * The Fonseca Fleming function N.1 is defined by
+ *
+ * \f[
+ * f_{1}\left(\boldsymbol{x}\right) = 1 - \exp \left[-\sum_{i=1}^{3} \left(x_{i} - \frac{1}{\sqrt{n}} \right)^{2} \right] \\
+ * f_{2}\left(\boldsymbol{x}\right) = 1 - \exp \left[-\sum_{i=1}^{3} \left(x_{i} + \frac{1}{\sqrt{n}} \right)^{2} \right] \\
+ * \f]
+ *
+ * The optimal solutions to this multi-objective function lie in the
+ * range [-1/sqrt(3), 1/sqrt(3)].
+ *
+ * @tparam arma::mat Type of matrix to optimize.
+ */
+template<typename MatType = arma::mat>
+class FonsecaFlemingFunction
+{
+ private:
+  size_t numObjectives;
+  size_t numVariables;
+
+ public:
+  FonsecaFlemingFunction() : numObjectives(2), numVariables(3)
+  {/* Nothing to do here. */}
+
+  /**
+   * Evaluate the objectives with the given coordinate.
+   *
+   * @param coords The function coordinates.
+   * @return arma::Col<typename MatType::elem_type>
+   */
+  arma::Col<typename MatType::elem_type> Evaluate(const MatType& coords)
+  {
+    // Convenience typedef.
+    typedef typename MatType::elem_type ElemType;
+
+    arma::Col<ElemType> objectives(numObjectives);
+
+    objectives(0) = objectiveA.Evaluate(coords);
+    objectives(1) = objectiveB.Evaluate(coords);
+
+    return objectives;
+  }
+
+  //! Get the starting point.
+  MatType GetInitialPoint()
+  {
+    // Convenience typedef.
+    typedef typename MatType::elem_type ElemType;
+
+    return arma::Col<ElemType>(numVariables, 1, arma::fill::zeros);
+  }
+
+  struct ObjectiveA
+  {
+    typename MatType::elem_type Evaluate(const MatType& coords)
+    {
+        return 1.0 - exp(
+             -pow(static_cast<double>(coords[0]) - 1.0 / sqrt(3.0), 2.0)
+             -pow(static_cast<double>(coords[1]) - 1.0 / sqrt(3.0), 2.0)
+             -pow(static_cast<double>(coords[2]) - 1.0 / sqrt(3.0), 2.0)
+        );
+    }
+  } objectiveA;
+
+  struct ObjectiveB
+  {
+    typename MatType::elem_type Evaluate(const MatType& coords)
+    {
+        return 1.0 - exp(
+            -pow(static_cast<double>(coords[0]) + 1.0 / sqrt(3.0), 2.0)
+            -pow(static_cast<double>(coords[1]) + 1.0 / sqrt(3.0), 2.0)
+            -pow(static_cast<double>(coords[2]) + 1.0 / sqrt(3.0), 2.0)
+        );
+    }
+  } objectiveB;
+
+  //! Get objective functions.
+  std::tuple<ObjectiveA, ObjectiveB> GetObjectives()
+  {
+    return std::make_tuple(objectiveA, objectiveB);
+  }
+};
+} // namespace test
+} // namespace ens
+
+#endif

include/ensmallen_bits/problems/generalized_rosenbrock_function.hpp

@@ -61,13 +61,6 @@ class GeneralizedRosenbrockFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return n - 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  const MatType GetInitialPoint() const
-  {
-    return arma::conv_to<MatType>::from(initialPoint);
-  }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -111,6 +104,28 @@ class GeneralizedRosenbrockFunction
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient) const;
 
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  const MatType GetInitialPoint() const
+  {
+    return arma::conv_to<MatType>::from(initialPoint);
+  }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  const MatType GetFinalPoint() const
+  {
+    return arma::ones<MatType>(initialPoint.n_rows, initialPoint.n_cols);
+  }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 0.0; }
+
  private:
   //! Locally-stored Initial point.
   arma::mat initialPoint;

include/ensmallen_bits/problems/goldstein_price_function.hpp

@@ -61,10 +61,6 @@ class GoldsteinPriceFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const { return MatType("-2; 2"); }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -107,6 +103,22 @@ class GoldsteinPriceFunction
    */
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient);
+
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("0.2; -0.5"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("0.0; -1.0"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 3.0; }
 };
 
 } // namespace test

include/ensmallen_bits/problems/gradient_descent_test_function.hpp

@@ -25,10 +25,6 @@ class GDTestFunction
   //! Nothing to do for the constructor.
   GDTestFunction() { }
 
-  //! Get the starting point.
-  template<typename MatType>
-  MatType GetInitialPoint() const { return MatType("1; 3; 2"); }
-
   //! Evaluate a function.
   template<typename MatType>
   typename MatType::elem_type Evaluate(const MatType& coordinates) const;
@@ -36,6 +32,22 @@ class GDTestFunction
   //! Evaluate the gradient of a function.
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient) const;
+
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType>
+  MatType GetInitialPoint() const { return MatType("1; 3; 2"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("0; 0; 0"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 0.0; }
 };
 
 } // namespace test

include/ensmallen_bits/problems/levy_function_n13.hpp

@@ -48,10 +48,6 @@ class LevyFunctionN13
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const { return MatType("-10; 10"); }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -94,6 +90,22 @@ class LevyFunctionN13
    */
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient);
+
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("0.9; 1.1"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("1.0; 1.0"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 0.0; }
 };
 
 } // namespace test

include/ensmallen_bits/problems/matyas_function.hpp

@@ -52,10 +52,6 @@ class MatyasFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const { return MatType("-3; 3"); }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -98,6 +94,22 @@ class MatyasFunction
    */
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient);
+
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("-3; 3"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("0.0; 0.0"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 0.0; }
 };
 
 } // namespace test

include/ensmallen_bits/problems/mc_cormick_function.hpp

@@ -52,10 +52,6 @@ class McCormickFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const { return MatType("-2; 4"); }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -98,6 +94,22 @@ class McCormickFunction
    */
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient);
+
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("-2; 4"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("-0.54719; -1.54719"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return -1.9133; }
 };
 
 } // namespace test

include/ensmallen_bits/problems/problems.hpp

@@ -17,6 +17,7 @@
 #include "drop_wave_function.hpp"
 #include "easom_function.hpp"
 #include "eggholder_function.hpp"
+#include "fonseca_fleming_function.hpp"
 #include "fw_test_function.hpp"
 #include "generalized_rosenbrock_function.hpp"
 #include "goldstein_price_function.hpp"
@@ -30,6 +31,7 @@
 #include "rastrigin_function.hpp"
 #include "rosenbrock_function.hpp"
 #include "rosenbrock_wood_function.hpp"
+#include "schaffer_function_n1.hpp"
 #include "schaffer_function_n2.hpp"
 #include "schaffer_function_n4.hpp"
 #include "schwefel_function.hpp"
@@ -40,5 +42,10 @@
 #include "styblinski_tang_function.hpp"
 #include "three_hump_camel_function.hpp"
 #include "wood_function.hpp"
+#include "zdt/zdt1_function.hpp"
+#include "zdt/zdt2_function.hpp"
+#include "zdt/zdt3_function.hpp"
+#include "zdt/zdt4_function.hpp"
+#include "zdt/zdt6_function.hpp"
 
 #endif

include/ensmallen_bits/problems/rastrigin_function.hpp

@@ -44,24 +44,17 @@ class RastriginFunction
    *
    * @param n Number of dimensions for the function.
    */
-  RastriginFunction(const size_t n);
+  RastriginFunction(const size_t n = 2);
 
   /**
    * Shuffle the order of function visitation. This may be called by the
    * optimizer.
    */
- void Shuffle();
+  void Shuffle();
 
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return n; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const
-  {
-    return arma::conv_to<MatType>::from(initialPoint);
-  }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -104,6 +97,29 @@ class RastriginFunction
    */
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient);
+
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const
+  {
+    return arma::conv_to<MatType>::from(initialPoint);
+  }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const
+  {
+    return arma::zeros<MatType>(initialPoint.n_rows, initialPoint.n_cols);
+  }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 0.0; }
+
  private:
   //! Number of dimensions for the function.
   size_t n;

include/ensmallen_bits/problems/rosenbrock_function.hpp

@@ -55,16 +55,6 @@ class RosenbrockFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const
-  {
-    MatType m(2, 1);
-    m[0] = -1.2;
-    m[1] = 1.0;
-    return m;
-  }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -114,6 +104,22 @@ class RosenbrockFunction
   template<typename MatType, typename GradType>
   typename MatType::elem_type EvaluateWithGradient(const MatType& coordinates,
                                                    GradType& gradient) const;
+
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("-1.2; 1.0"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("1.0; 1.0"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 0.0; }
 };
 
 } // namespace test

include/ensmallen_bits/problems/rosenbrock_wood_function.hpp

@@ -39,13 +39,6 @@ class RosenbrockWoodFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  const MatType GetInitialPoint() const
-  {
-    return arma::conv_to<MatType>::from(initialPoint);
-  }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -89,6 +82,28 @@ class RosenbrockWoodFunction
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient) const;
 
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  const MatType GetInitialPoint() const
+  {
+    return arma::conv_to<MatType>::from(initialPoint);
+  }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const
+  {
+    return arma::ones<MatType>(initialPoint.n_rows, initialPoint.n_cols);
+  }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 0.0; }
+
  private:
   //! Locally-stored initial point.
   arma::mat initialPoint;

include/ensmallen_bits/problems/schaffer_function_n1.hpp

@@ -0,0 +1,97 @@
+/**
+ * @file schaffer_function_n1.hpp
+ * @author Sayan Goswami
+ *
+ * Implementation of Schaffer function N.1.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_SCHAFFER_FUNCTION_N1_HPP
+#define ENSMALLEN_PROBLEMS_SCHAFFER_FUNCTION_N1_HPP
+
+namespace ens {
+namespace test {
+
+/**
+ * The Schaffer function N.1 is defined by
+ *
+ * \f[
+ * f_1(x) = x^2
+ * f_2(x) = (x-2)^2
+ * \f]
+ *
+ * The optimal solutions to this multi-objective function lie in the
+ * range [0, 2].
+ *
+ * @tparam arma::mat Type of matrix to optimize.
+ */
+template<typename MatType = arma::mat>
+class SchafferFunctionN1
+{
+ private:
+  size_t numObjectives;
+  size_t numVariables;
+
+ public:
+ //! Initialize the SchafferFunctionN1
+  SchafferFunctionN1() : numObjectives(2), numVariables(1)
+  {/* Nothing to do here. */}
+
+  /**
+   * Evaluate the objectives with the given coordinate.
+   *
+   * @param coords The function coordinates.
+   * @return arma::Col<typename MatType::elem_type>
+   */
+  arma::Col<typename MatType::elem_type> Evaluate(const MatType& coords)
+  {
+    // Convenience typedef.
+    typedef typename MatType::elem_type ElemType;
+
+    arma::Col<ElemType> objectives(numObjectives);
+
+    objectives(0) = std::pow(coords[0], 2);
+    objectives(1) = std::pow(coords[0] - 2, 2);
+
+    return objectives;
+  }
+
+  //! Get the starting point.
+  MatType GetInitialPoint()
+  {
+    // Convenience typedef.
+    typedef typename MatType::elem_type ElemType;
+
+    return arma::Col<ElemType>(numVariables, 1, arma::fill::zeros);
+  }
+
+  struct ObjectiveA
+  {
+    typename MatType::elem_type Evaluate(const MatType& coords)
+    {
+        return std::pow(coords[0], 2);
+    }
+  } objectiveA;
+
+  struct ObjectiveB
+  {
+    typename MatType::elem_type Evaluate(const MatType& coords)
+    {
+        return std::pow(coords[0] - 2, 2);
+    }
+  } objectiveB;
+
+  //! Get objective functions.
+  std::tuple<ObjectiveA, ObjectiveB> GetObjectives()
+  {
+    return std::make_tuple(objectiveA, objectiveB);
+  }
+};
+} // namespace test
+} // namespace ens
+
+#endif

include/ensmallen_bits/problems/schaffer_function_n2.hpp

@@ -41,7 +41,7 @@ namespace test {
 class SchafferFunctionN2
 {
  public:
-  //! Initialize the SchafferFunctionN4.
+  //! Initialize the SchafferFunctionN2.
   SchafferFunctionN2();
 
   /**
@@ -53,10 +53,6 @@ class SchafferFunctionN2
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const { return MatType("-100; 100"); }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -99,6 +95,22 @@ class SchafferFunctionN2
    */
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient);
+
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("-100; 100"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("0.0; 0.0"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 0.0; }
 };
 
 } // namespace test

include/ensmallen_bits/problems/schwefel_function.hpp

@@ -55,13 +55,6 @@ class SchwefelFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return n; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const
-  {
-    return arma::conv_to<MatType>::from(initialPoint);
-  }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -104,6 +97,31 @@ class SchwefelFunction
    */
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient);
+
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const
+  {
+    return arma::conv_to<MatType>::from(initialPoint);
+  }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const
+  {
+    MatType result(initialPoint.n_rows, initialPoint.n_cols, arma::fill::none);
+    result.fill(420.9687);
+    return result;
+  }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 0.0; }
+
  private:
   //! Number of dimensions for the function.
   size_t n;

include/ensmallen_bits/problems/sgd_test_function.hpp

@@ -37,10 +37,6 @@ class SGDTestFunction
   //! Return 3 (the number of functions).
   size_t NumFunctions() const { return 3; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const { return MatType("6; -45.6; 6.2"); }
-
   //! Evaluate a function for a particular batch-size.
   template<typename MatType>
   typename MatType::elem_type Evaluate(const MatType& coordinates,
@@ -53,6 +49,22 @@ class SGDTestFunction
                 const size_t begin,
                 GradType& gradient,
                 const size_t batchSize) const;
+
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("6; -45.6; 6.2"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("0.0; 0.0; 0.0"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return -1.0; }
 };
 
 } // namespace test

include/ensmallen_bits/problems/sparse_test_function.hpp

@@ -32,10 +32,6 @@ class SparseTestFunction
   //! Return 4 (the number of features).
   size_t NumFeatures() const { return 4; }
 
-  //! Get the starting point.
-  template<typename MatType>
-  MatType GetInitialPoint() const { return MatType("0 0 0 0;"); }
-
   //! Evaluate a function.
   template<typename MatType>
   typename MatType::elem_type Evaluate(const MatType& coordinates,
@@ -61,6 +57,22 @@ class SparseTestFunction
                        const size_t j,
                        GradType& gradient) const;
 
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType>
+  MatType GetInitialPoint() const { return MatType("0 0 0 0;"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("2.0 1.0 1.5 4.0"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 123.75; }
+
  private:
   // Each quadratic polynomial is monic. The intercept and coefficient of the
   // first order term is stored.

include/ensmallen_bits/problems/sphere_function.hpp

@@ -45,7 +45,7 @@ class SphereFunction
    *
    * @param n Number of dimensions for the function.
    */
-  SphereFunction(const size_t n);
+  SphereFunction(const size_t n = 2);
 
   /**
    * Shuffle the order of function visitation. This may be called by the
@@ -56,13 +56,6 @@ class SphereFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return n; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const
-  {
-    return arma::conv_to<MatType>::from(initialPoint);
-  }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -106,6 +99,28 @@ class SphereFunction
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient);
 
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const
+  {
+    return arma::conv_to<MatType>::from(initialPoint);
+  }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const
+  {
+    return arma::zeros<MatType>(initialPoint.n_rows, initialPoint.n_cols);
+  }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 0.0; }
+
  private:
   //! Number of dimensions for the function.
   size_t n;

include/ensmallen_bits/problems/styblinski_tang_function.hpp

@@ -46,7 +46,7 @@ class StyblinskiTangFunction
    *
    * @param n Number of dimensions for the function.
    */
-  StyblinskiTangFunction(const size_t n);
+  StyblinskiTangFunction(const size_t n = 2);
 
   /**
    * Shuffle the order of function visitation. This may be called by the
@@ -57,13 +57,6 @@ class StyblinskiTangFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return n; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const
-  {
-    return arma::conv_to<MatType>::from(initialPoint);
-  }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -107,6 +100,31 @@ class StyblinskiTangFunction
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient);
 
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const
+  {
+    return arma::conv_to<MatType>::from(initialPoint);
+  }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const
+  {
+    MatType result(initialPoint.n_rows, initialPoint.n_cols);
+    for (size_t i = 0; i < result.n_elem; ++i)
+      result[i] = -2.903534;
+    return result;
+  }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return -39.16599 * n; }
+
  private:
   //! Number of dimensions for the function.
   size_t n;

include/ensmallen_bits/problems/three_hump_camel_function.hpp

@@ -58,10 +58,6 @@ class ThreeHumpCamelFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const { return MatType("-4.5; 4.5"); }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -104,6 +100,22 @@ class ThreeHumpCamelFunction
    */
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient);
+
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("1; 1"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("0.0; 0.0"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 0.0; }
 };
 
 } // namespace test

include/ensmallen_bits/problems/wood_function.hpp

@@ -59,10 +59,6 @@ class WoodFunction
   //! Return 1 (the number of functions).
   size_t NumFunctions() const { return 1; }
 
-  //! Get the starting point.
-  template<typename MatType = arma::mat>
-  MatType GetInitialPoint() const { return MatType("-3; -1; -3; -1"); }
-
   /**
    * Evaluate a function for a particular batch-size.
    *
@@ -105,6 +101,22 @@ class WoodFunction
    */
   template<typename MatType, typename GradType>
   void Gradient(const MatType& coordinates, GradType& gradient) const;
+
+  // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not
+  // required for using ensmallen to optimize this function!  They are
+  // specifically used as a convenience just for ensmallen's testing
+  // infrastructure.
+
+  //! Get the starting point.
+  template<typename MatType = arma::mat>
+  MatType GetInitialPoint() const { return MatType("-3; -1; -3; -1"); }
+
+  //! Get the final point.
+  template<typename MatType = arma::mat>
+  MatType GetFinalPoint() const { return MatType("1; 1; 1; 1"); }
+
+  //! Get the final objective.
+  double GetFinalObjective() const { return 0.0; }
 };
 
 } // namespace test

include/ensmallen_bits/problems/zdt/zdt1_function.hpp

@@ -0,0 +1,157 @@
+/**
+ * @file zdt1_function.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * Implementation of the first ZDT(Zitzler, Deb and Thiele) test.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_ZDT_ONE_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_ZDT_ONE_FUNCTION_HPP
+
+namespace ens {
+namespace test {
+
+/**
+ * The ZDT1 function, defined by:
+ * \f[
+ * g(x) = 1 + 9(\sum_{i=2}^{n} x_i )/(n-1)
+ * f_1(x) = x_1
+ * h(f_1, g) = g(x)[1-\sqrt{f_1/g}\ ]
+ * f_2(x) = g(x) * h(f_1, g)
+ * \f]
+ *
+ * This is a 30-variable problem (n = 30) with a convex optimal set.
+ *
+ * Bounds of the variable space is:
+ * 0 <= x_i <= 1 for i = 1,...,n.
+ *
+ * This should be optimized to g(x) = 1.0, at:
+ * x_1* in [0, 1] ; x_i* = 0 for i = 2,...,n
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Zitzler2000,
+ *   title   = {Comparison of multiobjective evolutionary algorithms:
+ *              Empirical results},
+ *   author  = {Zitzler, Eckart and Deb, Kalyanmoy and Thiele, Lothar},
+ *   journal = {Evolutionary computation},
+ *   year    = {2000},
+ *   doi     = {10.1162/106365600568202}
+ * }
+ * @endcode
+ *
+ * @tparam MatType Type of matrix to optimize.
+ */
+  template<typename MatType = arma::mat>
+  class ZDT1
+  {
+   private:
+    size_t numParetoPoints {100};
+    size_t numObjectives {2};
+    size_t numVariables {30};
+
+   public:
+     //! Initialize the ZDT1
+    ZDT1(size_t numParetoPoints = 100) :
+        numParetoPoints(numParetoPoints),
+        objectiveF1(*this),
+        objectiveF2(*this)
+    {/* Nothing to do here. */}
+
+    /**
+     * Evaluate the objectives with the given coordinate.
+     *
+     * @param coords The function coordinates.
+     * @return arma::Col<typename MatType::elem_type>
+     */
+    arma::Col<typename MatType::elem_type> Evaluate(const MatType& coords)
+    {
+      // Convenience typedef.
+      typedef typename MatType::elem_type ElemType;
+
+      arma::Col<ElemType> objectives(numObjectives);
+      objectives(0) = coords[0];
+      ElemType sum = arma::accu(coords(arma::span(1, numVariables - 1), 0));
+      ElemType g = 1. + 9. * sum / (static_cast<ElemType>(numVariables) - 1.);
+      ElemType objectiveRatio = objectives(0) / g;
+      objectives(1) = g * (1. - std::sqrt(objectiveRatio));
+
+      return objectives;
+    }
+
+    //! Get the starting point.
+    MatType GetInitialPoint()
+    {
+      // Convenience typedef.
+      typedef typename MatType::elem_type ElemType;
+
+      return arma::Col<ElemType>(numVariables, 1, arma::fill::zeros);
+    }
+
+    struct ObjectiveF1
+    {
+      ObjectiveF1(ZDT1& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        return coords[0];
+      }
+
+      ZDT1& zdtClass;
+    };
+
+    struct ObjectiveF2
+    {
+      ObjectiveF2(ZDT1& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+
+        size_t numVariables = zdtClass.numVariables;
+        ElemType sum = arma::accu(coords(arma::span(1, numVariables - 1), 0));
+        ElemType g = 1. + 9. * sum / (static_cast<ElemType>(numVariables - 1));
+        ElemType objectiveRatio = zdtClass.objectiveF1.Evaluate(coords) / g;
+
+        return g * (1. - std::sqrt(objectiveRatio));
+      }
+
+      ZDT1& zdtClass;
+    };
+
+    //! Get objective functions.
+    std::tuple<ObjectiveF1, ObjectiveF2> GetObjectives()
+    {
+      return std::make_tuple(objectiveF1, objectiveF2);
+    }
+
+    //! Get the Reference Front.
+    //! Refer PR #273 Ipynb notebook to see the plot of Reference
+    //! Front. The implementation has been taken from pymoo.
+    arma::cube GetReferenceFront()
+    {
+      arma::cube front(2, 1, numParetoPoints);
+      arma::vec x = arma::linspace(0, 1, numParetoPoints);
+      arma::vec y = 1 - arma::sqrt(x);
+      for (size_t idx = 0; idx < numParetoPoints; ++idx)
+        front.slice(idx) = arma::vec{ x(idx), y(idx) };
+
+      return front;
+    }
+
+    ObjectiveF1 objectiveF1;
+    ObjectiveF2 objectiveF2;
+  };
+  } //namespace test
+  } //namespace ens
+
+#endif

include/ensmallen_bits/problems/zdt/zdt2_function.hpp

@@ -0,0 +1,158 @@
+/**
+ * @file zdt2_function.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * Implementation of the second ZDT(Zitzler, Deb and Thiele) test.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_ZDT_TWO_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_ZDT_TWO_FUNCTION_HPP
+
+namespace ens {
+namespace test {
+
+/**
+ * The ZDT2 function, defined by:
+ * \f[
+ * g(x) = 1 + 9(\sum_{i=2}^{n} x_i )/(n-1)
+ * f_1(x) = x_1
+ * h(f_1, g) = 1 - (f_1/g)^2
+ * f_2(x) = g(x) * h(f_1, g)
+ * \f]
+ *
+ * This is a 30-variable problem(n = 30) with a
+ * non convex optimal set.
+ *
+ * Bounds of the variable space is:
+ * 0 <= x_i <= 1 for i = 1,..,n.
+ *
+ * This should be optimized to g(x) = 1.0, at:
+ * x_1* in [0, 1] ; x_i* = 0 for i = 2,...,n
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Zitzler2000,
+ *   title   = {Comparison of multiobjective evolutionary algorithms:
+ *              Empirical results},
+ *   author  = {Zitzler, Eckart and Deb, Kalyanmoy and Thiele, Lothar},
+ *   journal = {Evolutionary computation},
+ *   year    = {2000},
+ *   doi     = {10.1162/106365600568202}
+ * }
+ * @endcode
+ *
+ * @tparam MatType Type of matrix to optimize.
+ */
+  template<typename MatType = arma::mat>
+  class ZDT2
+  {
+   private:
+    size_t numParetoPoints {100};
+    size_t numObjectives {2};
+    size_t numVariables {30};
+
+   public:
+     //! Initialize the ZDT2
+    ZDT2(size_t numParetoPoints = 100) :
+        numParetoPoints(numParetoPoints),
+        objectiveF1(*this),
+        objectiveF2(*this)
+    {/* Nothing to do here. */}
+
+    /**
+     * Evaluate the objectives with the given coordinate.
+     *
+     * @param coords The function coordinates.
+     * @return arma::Col<typename MatType::elem_type>
+     */
+    arma::Col<typename MatType::elem_type> Evaluate(const MatType& coords)
+    {
+      // Convenience typedef.
+      typedef typename MatType::elem_type ElemType;
+
+      arma::Col<ElemType> objectives(numObjectives);
+      objectives(0) = coords[0];
+      ElemType sum = arma::accu(coords(arma::span(1, numVariables - 1), 0));
+      ElemType g = 1. + 9. * sum / (static_cast<ElemType>(numVariables) - 1.);
+      ElemType objectiveRatio = objectives(0) / g;
+      objectives(1) = g * (1. - std::pow(objectiveRatio, 2));
+
+      return objectives;
+    }
+
+    //! Get the starting point.
+    MatType GetInitialPoint()
+    {
+      // Convenience typedef.
+      typedef typename MatType::elem_type ElemType;
+
+      return arma::Col<ElemType>(numVariables, 1, arma::fill::zeros);
+    }
+
+    struct ObjectiveF1
+    {
+      ObjectiveF1(ZDT2& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        return coords[0];
+      }
+
+      ZDT2& zdtClass;
+    };
+
+    struct ObjectiveF2
+    {
+      ObjectiveF2(ZDT2& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        // Convenience typedef.
+        typedef typename MatType::elem_type ElemType;
+
+        size_t numVariables = zdtClass.numVariables;
+        ElemType sum = arma::accu(coords(arma::span(1, numVariables - 1), 0));
+        ElemType g = 1. + 9. * sum / (static_cast<ElemType>(numVariables - 1));
+        ElemType objectiveRatio = zdtClass.objectiveF1.Evaluate(coords) / g;
+
+        return g * (1. - std::pow(objectiveRatio, 2));
+      }
+
+      ZDT2& zdtClass;
+    };
+
+    //! Get objective functions.
+    std::tuple<ObjectiveF1, ObjectiveF2> GetObjectives()
+    {
+      return std::make_tuple(objectiveF1, objectiveF2);
+    }
+
+    //! Get the Reference Front.
+    //! Refer PR #273 Ipynb notebook to see the plot of Reference
+    //! Front. The implementation has been taken from pymoo.
+    arma::cube GetReferenceFront()
+    {
+      arma::cube front(2, 1, numParetoPoints);
+      arma::vec x = arma::linspace(0, 1, numParetoPoints);
+      arma::vec y = 1 - arma::square(x);
+      for (size_t idx = 0; idx < numParetoPoints; ++idx)
+        front.slice(idx) = arma::vec{ x(idx), y(idx) };
+
+      return front;
+    }
+
+    ObjectiveF1 objectiveF1;
+    ObjectiveF2 objectiveF2;
+  };
+  } //namespace test
+  } //namespace ens
+
+#endif

include/ensmallen_bits/problems/zdt/zdt3_function.hpp

@@ -0,0 +1,188 @@
+/**
+ * @file zdt3_function.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * Implementation of the third ZDT(Zitzler, Deb and Thiele) test.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_ZDT_THREE_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_ZDT_THREE_FUNCTION_HPP
+
+namespace ens {
+namespace test {
+
+/**
+ * The ZDT3 function, defined by:
+ * \f[
+ * g(x) = 1 + 9(\sum_{i=2}^{n} x_i )/(n-1)
+ * f_1(x) = x_1
+ * h(f_1,g) = 1 - \sqrt{f_1/g}  - (f_1/g)sin(10\pi f_1)
+ * f_2(x) = g(x) * h(f_1, g)
+ * \f]
+ *
+ * This is a 30-variable problem(n = 30) with a number
+ * of disconnected optimal fronts.
+ *
+ * Bounds of the variable space is:
+ * 0 <= x_i <= 1 for i = 1,...,n.
+ *
+ * This should be optimized to g(x) = 1.0, at:
+ *
+ * x_1* in [0.0000, 0.0830] OR
+ * x_1* in [0.1822, 0.2577] OR
+ * x_1* in [0.4093, 0.4538] OR
+ * x_1* in [0.6183, 0.6525] OR
+ * x_1* in [0.8233, 0.8518].
+ *
+ * x_i* = 0 for i = 2,...,n.
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Zitzler2000,
+ *   title   = {Comparison of multiobjective evolutionary algorithms:
+ *              Empirical results},
+ *   author  = {Zitzler, Eckart and Deb, Kalyanmoy and Thiele, Lothar},
+ *   journal = {Evolutionary computation},
+ *   year    = {2000},
+ *   doi     = {10.1162/106365600568202}
+ * }
+ * @endcode
+ *
+ * @tparam MatType Type of matrix to optimize.
+ */
+  template<typename MatType = arma::mat>
+  class ZDT3
+  {
+   private:
+    size_t numParetoPoints {100};
+    size_t numObjectives {2};
+    size_t numVariables {30};
+
+   public:
+     //! Initialize the ZDT3
+    ZDT3(size_t numParetoPoints = 100) :
+        numParetoPoints(numParetoPoints),
+        objectiveF1(*this),
+        objectiveF2(*this)
+    {/* Nothing to do here. */}
+
+    /**
+     * Evaluate the objectives with the given coordinate.
+     *
+     * @param coords The function coordinates.
+     * @return arma::Col<typename MatType::elem_type>
+     */
+    arma::Col<typename MatType::elem_type> Evaluate(const MatType& coords)
+    {
+      typedef typename MatType::elem_type ElemType;
+
+      arma::Col<ElemType> objectives(numObjectives);
+      objectives(0) = coords[0];
+      ElemType sum = arma::accu(coords(arma::span(1, numVariables - 1), 0));
+      ElemType g = 1. + 9. * sum / (static_cast<ElemType>(numVariables) - 1.);
+      ElemType objectiveRatio = objectives(0) / g;
+      objectives(1) = g * (1. - std::sqrt(objectiveRatio) -
+          (objectiveRatio) * std::sin(10. * arma::datum::pi * coords[0]));
+
+      return objectives;
+    }
+
+    //! Get the starting point.
+    MatType GetInitialPoint()
+    {
+      // Convenience typedef.
+      typedef typename MatType::elem_type ElemType;
+
+      return arma::Col<ElemType>(numVariables, 1, arma::fill::zeros);
+    }
+
+    struct ObjectiveF1
+    {
+      ObjectiveF1(ZDT3& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        return coords[0];
+      }
+
+      ZDT3& zdtClass;
+    };
+
+    struct ObjectiveF2
+    {
+      ObjectiveF2(ZDT3& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        typedef typename MatType::elem_type ElemType;
+
+        size_t numVariables = zdtClass.numVariables;
+        ElemType sum = arma::accu(coords(arma::span(1, numVariables - 1), 0));
+        ElemType g = 1. + 9. * sum / (static_cast<ElemType>(numVariables - 1));
+        ElemType objectiveRatio = zdtClass.objectiveF1.Evaluate(coords) / g;
+
+        return g * (1. - std::sqrt(objectiveRatio) -
+            (objectiveRatio) * std::sin(10. * arma::datum::pi * coords[0]));
+      }
+
+      ZDT3& zdtClass;
+    };
+
+    //! Get objective functions.
+    std::tuple<ObjectiveF1, ObjectiveF2> GetObjectives()
+    {
+      return std::make_tuple(objectiveF1, objectiveF2);
+    }
+
+    //! Get the Reference Front.
+    //! Refer PR #273 Ipynb notebook to see the plot of Reference
+    //! Front. The implementation has been taken from pymoo.
+    arma::cube GetReferenceFront()
+    {
+      size_t numRegions = 5;
+      size_t regionDensity = std::floor(numParetoPoints / numRegions);
+      size_t apparentParetoPoints = numRegions * regionDensity;
+      arma::cube front(2, 1, apparentParetoPoints);
+
+      arma::mat regions{
+        {0.0, 0.182228780, 0.4093136748,
+         0.6183967944, 0.8233317983},
+        {0.0830015349, 0.2577623634, 0.4538821041,
+         0.6525117038, 0.8518328654}
+      };
+
+      for (size_t regionIdx = 0; regionIdx < numRegions; ++regionIdx)
+      {
+        arma::vec region = regions.col(regionIdx);
+        //! Generate x and y coordinates for the region.
+        arma::vec x = arma::linspace(
+            region(0), region(1), regionDensity);
+        arma::vec y = 1 - arma::sqrt(x) - x
+            % arma::sin(10 * arma::datum::pi * x);
+
+        //! Fill the front with the generated points.
+        for (size_t pointIdx = 0; pointIdx < regionDensity; ++pointIdx)
+        {
+          size_t sliceIdx = regionIdx * regionDensity + pointIdx;
+          front.slice(sliceIdx) = arma::vec{ x(pointIdx), y(pointIdx) };
+        }
+      }
+
+      return front;
+    }
+
+    ObjectiveF1 objectiveF1;
+    ObjectiveF2 objectiveF2;
+  };
+  } //namespace test
+  } //namespace ens
+
+#endif

include/ensmallen_bits/problems/zdt/zdt4_function.hpp

@@ -0,0 +1,160 @@
+/**
+ * @file zdt4_function.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * Implementation of the fourth ZDT(Zitzler, Deb and Thiele) test.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_ZDT_FOUR_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_ZDT_FOUR_FUNCTION_HPP
+
+namespace ens {
+namespace test {
+/**
+ * The ZDT4 function, defined by:
+ * \f[
+ * g(x) = 1 + 10(n-1) + \sum_{i=2}^{n}(x_i^2 - 10cos(4\pi x_i))
+ * f_1(x) = x_i
+ * h(f_1,g) = 1 - \sqrt{f_i/g}
+ * f_2(x) = g(x) * h(f_1, g)
+ * \f]
+ *
+ * This is a 10-variable problem(n = 10) with a convex
+ * optimal front. This problem contains several local
+ * optimum, making it difficult to reach the global optimum.
+ *
+ * Bounds of the variable space is:
+ *  0 <= x_1 <= 1;
+ * -10 <= x_i <= 10 for i = 2,...,n.
+ *
+ * This should be optimized to g(x) = 1.0, at:
+ * x_1* in [0, 1] ; x_i* = 0 for i = 2,...,n
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Zitzler2000,
+ *   title   = {Comparison of multiobjective evolutionary algorithms:
+ *              Empirical results},
+ *   author  = {Zitzler, Eckart and Deb, Kalyanmoy and Thiele, Lothar},
+ *   journal = {Evolutionary computation},
+ *   year    = {2000},
+ *   doi     = {10.1162/106365600568202}
+ * }
+ * @endcode
+ *
+ * @tparam MatType Type of matrix to optimize.
+ */
+  template<typename MatType = arma::mat>
+  class ZDT4
+  {
+   private:
+    size_t numParetoPoints {100};
+    size_t numObjectives {2};
+    size_t numVariables {10};
+
+   public:
+     //! Initialize the ZDT4
+    ZDT4(size_t numParetoPoints = 100) :
+        numParetoPoints(numParetoPoints),
+        objectiveF1(*this),
+        objectiveF2(*this)
+    {/* Nothing to do here. */}
+
+    /**
+     * Evaluate the objectives with the given coordinate.
+     *
+     * @param coords The function coordinates.
+     * @return arma::Col<typename MatType::elem_type>
+     */
+    arma::Col<typename MatType::elem_type> Evaluate(const MatType& coords)
+    {
+      typedef typename MatType::elem_type ElemType;
+
+      arma::Col<ElemType> objectives(numObjectives);
+      objectives(0) = coords[0];
+      MatType truncatedCoords = coords(arma::span(1, numVariables - 1), 0);
+      ElemType sum = arma::accu(arma::square(truncatedCoords) -
+          10. * arma::cos(4 * arma::datum::pi * truncatedCoords));
+      ElemType g = 1. + 10. * static_cast<ElemType>(numVariables - 1) + sum;
+      ElemType objectiveRatio = objectives(0) / g;
+	    objectives(1) = g * (1. - std::sqrt(objectiveRatio));
+
+      return objectives;
+    }
+
+    //! Get the starting point.
+    MatType GetInitialPoint()
+    {
+      // Convenience typedef.
+      typedef typename MatType::elem_type ElemType;
+
+      return arma::Col<ElemType>(numVariables, 1, arma::fill::zeros);
+    }
+
+    struct ObjectiveF1
+    {
+      ObjectiveF1(ZDT4& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        return coords[0];
+      }
+
+      ZDT4& zdtClass;
+    };
+
+    struct ObjectiveF2
+    {
+      ObjectiveF2(ZDT4& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        typedef typename MatType::elem_type ElemType;
+
+        size_t numVariables = zdtClass.numVariables;
+        MatType truncatedCoords = coords(arma::span(1, numVariables - 1), 0);
+        ElemType sum = arma::accu(arma::square(truncatedCoords) -
+            10. * arma::cos(4 * arma::datum::pi * truncatedCoords));
+        ElemType g = 1. + 10 * static_cast<ElemType>(numVariables - 1) + sum;
+        ElemType objectiveRatio = zdtClass.objectiveF1.Evaluate(coords) / g;
+
+        return  g * (1. - std::sqrt(objectiveRatio));
+      }
+
+      ZDT4& zdtClass;
+    };
+
+    //! Get objective functions.
+    std::tuple<ObjectiveF1, ObjectiveF2> GetObjectives()
+    {
+      return std::make_tuple(objectiveF1, objectiveF2);
+    }
+
+    //! Get the Reference Front.
+    //! Refer PR #273 Ipynb notebook to see the plot of Reference
+    //! Front. The implementation has been taken from pymoo.
+    arma::cube GetReferenceFront()
+    {
+      arma::cube front(2, 1, numParetoPoints);
+      arma::vec x = arma::linspace(0, 1, numParetoPoints);
+      arma::vec y = 1 - arma::sqrt(x);
+      for (size_t idx = 0; idx < numParetoPoints; ++idx)
+        front.slice(idx) = arma::vec{ x(idx), y(idx) };
+
+      return front;
+    }
+
+    ObjectiveF1 objectiveF1;
+    ObjectiveF2 objectiveF2;
+  };
+  } //namespace test
+  } //namespace ens
+#endif

include/ensmallen_bits/problems/zdt/zdt6_function.hpp

@@ -0,0 +1,162 @@
+/**
+ * @file zdt6_function.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * Implementation of the sixth ZDT(Zitzler, Deb and Thiele) test.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_PROBLEMS_ZDT_SIX_FUNCTION_HPP
+#define ENSMALLEN_PROBLEMS_ZDT_SIX_FUNCTION_HPP
+
+namespace ens {
+namespace test {
+/**
+ * The ZDT6 function, defined by:
+ * \f[
+ * g(x) = 1 + 9[ \sum_{i=2}^{n}(x_i^2)/9]^{0.25}
+ * f_1(x) = 1 - e^{-4x_1}sin^{6}(6\pi x_1)
+ * h(f1, g) = 1 - (f_1/g)^{2}
+ * f_2(x) = g(x) * h(f_1, g)
+ * \f]
+ *
+ * This is a 10-variable problem(n = 10) with a
+ * non-convex optimal front. The density of the
+ * solutions across optimal region is non-uniform.
+ *
+ * Bounds of the variable space is:
+ * 0 <= x_i <= 1 for i = 1,...,n
+ *
+ * This should be optimized to g(x) = 1.0, at:
+ * x_1* in [0, 1] ; x_i* = 0 for i = 2,...,n
+ *
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Zitzler2000,
+ *   title   = {Comparison of multiobjective evolutionary algorithms:
+ *              Empirical results},
+ *   author  = {Zitzler, Eckart and Deb, Kalyanmoy and Thiele, Lothar},
+ *   journal = {Evolutionary computation},
+ *   year    = {2000},
+ *   doi     = {10.1162/106365600568202}
+ * }
+ * @endcode
+ *
+ * @tparam MatType Type of matrix to optimize.
+ */
+  template<typename MatType = arma::mat>
+  class ZDT6
+  {
+   private:
+    size_t numParetoPoints {100};
+    size_t numObjectives {2};
+    size_t numVariables {10};
+
+   public:
+     //! Initialize the ZDT6
+    ZDT6(size_t numParetoPoints = 100) :
+        numParetoPoints(numParetoPoints),
+        objectiveF1(*this),
+        objectiveF2(*this)
+    {/* Nothing to do here. */}
+
+    /**
+     * Evaluate the objectives with the given coordinate.
+     *
+     * @param coords The function coordinates.
+     * @return arma::Col<typename MatType::elem_type>
+     */
+    arma::Col<typename MatType::elem_type> Evaluate(const MatType& coords)
+    {
+      // Convenience typedef.
+      typedef typename MatType::elem_type ElemType;
+
+      arma::Col<ElemType> objectives(numObjectives);
+      objectives(0) = 1. - std::exp(-4 * coords[0]) *
+          std::pow(std::sin(6 * arma::datum::pi * coords[0]), 6);
+      ElemType sum = std::pow(
+          arma::accu(coords(arma::span(1, numVariables - 1), 0)) / 9, 0.25);
+      ElemType g = 1. + 9. * sum;
+      ElemType objectiveRatio = objectives(0) / g;
+      objectives(1) = g * (1. - std::pow(objectiveRatio, 2));
+
+      return objectives;
+    }
+
+    //! Get the starting point.
+    MatType GetInitialPoint()
+    {
+      // Convenience typedef.
+      typedef typename MatType::elem_type ElemType;
+
+      return arma::Col<ElemType>(numVariables, 1, arma::fill::zeros);
+    }
+
+    struct ObjectiveF1
+    {
+      ObjectiveF1(ZDT6& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        return 1. - std::exp(-4. * coords[0]) *
+            std::pow(std::sin(6. * arma::datum::pi * coords[0]), 6.);
+      }
+
+      ZDT6& zdtClass;
+    };
+
+    struct ObjectiveF2
+    {
+      ObjectiveF2(ZDT6& zdtClass) : zdtClass(zdtClass)
+      {/*Nothing to do here */}
+
+      typename MatType::elem_type Evaluate(const MatType& coords)
+      {
+        typedef typename MatType::elem_type ElemType;
+
+        size_t numVariables = zdtClass.numVariables;
+
+        ElemType sum = std::pow(
+            arma::accu(coords(arma::span(1, numVariables - 1), 0)) / 9, 0.25);
+        ElemType g = 1. + 9. * sum;
+        ElemType objectiveRatio = zdtClass.objectiveF1.Evaluate(coords) / g;
+
+        return  g * (1. - std::pow(objectiveRatio, 2));
+    }
+
+      ZDT6& zdtClass;
+    };
+
+    //! Get objective functions.
+    std::tuple<ObjectiveF1, ObjectiveF2> GetObjectives()
+    {
+      return std::make_tuple(objectiveF1, objectiveF2);
+    }
+
+    //! Get the Reference Front.
+    //! Refer PR #273 Ipynb notebook to see the plot of Reference
+    //! Front. The implementation has been taken from pymoo.
+    arma::cube GetReferenceFront()
+    {
+      arma::cube front(2, 1, numParetoPoints);
+      arma::vec x = arma::linspace(0.2807753191, 1, numParetoPoints);
+      arma::vec y = 1 - arma::square(x);
+      for (size_t idx = 0; idx < numParetoPoints; ++idx)
+        front.slice(idx) = arma::vec{ x(idx), y(idx) };
+
+      return front;
+    }
+
+    ObjectiveF1 objectiveF1;
+    ObjectiveF2 objectiveF2;
+  };
+  } //namespace test
+  } //namespace ens
+#endif

include/ensmallen_bits/pso/pso.hpp

@@ -39,6 +39,7 @@ namespace ens {
  *
  * For more information, refer to:
  *
+ * @code
  * @inproceedings{Kennedy1995,
  *   author    = {Kennedy, James and Eberhart, Russell C.},
  *   booktitle = {Proceedings of the IEEE International Conference on
@@ -47,6 +48,7 @@ namespace ens {
  *   title     = {Particle swarm optimization},
  *   year      = 1995
  * }
+ * @endcode
  *
  * PSO can optimize arbitrary functions. For more details, see the documentation
  * on function types included with this distribution or on the ensmallen
@@ -108,6 +110,14 @@ class PSOType
           initPolicy(initPolicy)
   { /* Nothing to do. */ }
 
+  /**
+   * Clean memory associated with the PSO object.
+   */
+  ~PSOType()
+  {
+    instUpdatePolicy.Clean();
+  }
+
   /**
    * Construct the particle swarm optimizer with the given function and
    * parameters. The defaults here are not necessarily good for the given

include/ensmallen_bits/pso/pso_impl.hpp

@@ -18,7 +18,9 @@
 #include <queue>
 
 namespace ens {
-/* After the velocity of each particle is updated at the end of each iteration
+
+/**
+ * After the velocity of each particle is updated at the end of each iteration
  * in PSO, the position of particle i (in iteration j) is updated as:
  *
  * \f[

include/ensmallen_bits/pso/update_policies/lbest_update.hpp

@@ -73,7 +73,8 @@ class LBestUpdate
      *
      * @param parent Instantiated parent class.
      */
-     Policy(const LBestUpdate& /* parent */) { /* Do nothing. */ }
+     Policy(const LBestUpdate& /* parent */) : n(0)
+     { /* Do nothing. */ }
 
      /**
       * The Initialize method is called by PSO Optimizer method before the